]> git.ipfire.org Git - thirdparty/systemd.git/blame - src/libsystemd/sd-journal/compress.c
projects / thirdparty / systemd.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge pull request #18863 from keszybz/cmdline-escaping
[thirdparty/systemd.git] / src / libsystemd / sd-journal / compress.c
CommitLineData
db9ecf05 1/* SPDX-License-Identifier: LGPL-2.1-or-later */
e4e61fdb 2
82e24b00 3#include <inttypes.h>
e4e61fdb 4#include <stdlib.h>
4b5bc539 5#include <sys/mman.h>
ca78ad1d
ZJS		 6#include <sys/types.h>
7#include <sys/stat.h>
07630cea 8#include <unistd.h>
d89c8fdf 9
349cc4a5 10#if HAVE_XZ
3ffd4af2 11#include <lzma.h>
d89c8fdf
ZJS		 12#endif
13
349cc4a5 14#if HAVE_LZ4
3ffd4af2
LP		 15#include <lz4.h>
16#include <lz4frame.h>
d89c8fdf 17#endif
e4e61fdb 18
ef5924aa
NL		 19#if HAVE_ZSTD
20#include <zstd.h>
21#include <zstd_errors.h>
22#endif
23
b5efdb8a 24#include "alloc-util.h"
e4e61fdb 25#include "compress.h"
3ffd4af2 26#include "fd-util.h"
c004493c 27#include "io-util.h"
07630cea 28#include "journal-def.h"
355b59e2 29#include "macro.h"
d89c8fdf 30#include "sparse-endian.h"
8b43440b 31#include "string-table.h"
07630cea 32#include "string-util.h"
4094c4bf 33#include "unaligned.h"
07630cea 34#include "util.h"
d89c8fdf 35
349cc4a5 36#if HAVE_LZ4
fd421c4a
ZJS		 37DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_compressionContext_t, LZ4F_freeCompressionContext, NULL);
38DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext, NULL);
4b5bc539
ZJS		 39#endif
40
ef5924aa 41#if HAVE_ZSTD
fd421c4a
ZJS		 42DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_CCtx*, ZSTD_freeCCtx, NULL);
43DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_DCtx*, ZSTD_freeDCtx, NULL);
ef5924aa
NL		 44
45static int zstd_ret_to_errno(size_t ret) {
46 switch (ZSTD_getErrorCode(ret)) {
47 case ZSTD_error_dstSize_tooSmall:
48 return -ENOBUFS;
49 case ZSTD_error_memory_allocation:
50 return -ENOMEM;
51 default:
52 return -EBADMSG;
53 }
54}
55#endif
56
d89c8fdf
ZJS		 57#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
58
59static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
e9ece6a0
LP		 60 [OBJECT_COMPRESSED_XZ] = "XZ",
61 [OBJECT_COMPRESSED_LZ4] = "LZ4",
8653185a 62 [OBJECT_COMPRESSED_ZSTD] = "ZSTD",
e9ece6a0
LP		 63 /* If we add too many more entries here, it's going to grow quite large (and be mostly sparse), since
64 * the array key is actually a bitmask, not a plain enum */
d89c8fdf 65};
e4e61fdb 66
d89c8fdf
ZJS		 67DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
68
5d6f46b6
ZJS		 69int compress_blob_xz(const void *src, uint64_t src_size,
70 void *dst, size_t dst_alloc_size, size_t *dst_size) {
349cc4a5 71#if HAVE_XZ
1930eed2
JS		 72 static const lzma_options_lzma opt = {
73 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
0240c603
LP		 74 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
75 };
76 static const lzma_filter filters[] = {
77 { LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt },
78 { LZMA_VLI_UNKNOWN, NULL }
1930eed2 79 };
e4e61fdb 80 lzma_ret ret;
76cc0bf6 81 size_t out_pos = 0;
e4e61fdb
LP		 82
83 assert(src);
84 assert(src_size > 0);
85 assert(dst);
5d6f46b6 86 assert(dst_alloc_size > 0);
e4e61fdb
LP		 87 assert(dst_size);
88
d89c8fdf 89 /* Returns < 0 if we couldn't compress the data or the
e4e61fdb
LP		 90 * compressed result is longer than the original */
91
1930eed2
JS		 92 if (src_size < 80)
93 return -ENOBUFS;
94
95 ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
5d6f46b6 96 src, src_size, dst, &out_pos, dst_alloc_size);
e4e61fdb 97 if (ret != LZMA_OK)
d89c8fdf 98 return -ENOBUFS;
e4e61fdb 99
76cc0bf6 100 *dst_size = out_pos;
d89c8fdf
ZJS		 101 return 0;
102#else
103 return -EPROTONOSUPPORT;
104#endif
e4e61fdb
LP		 105}
106
5d6f46b6
ZJS		 107int compress_blob_lz4(const void *src, uint64_t src_size,
108 void *dst, size_t dst_alloc_size, size_t *dst_size) {
349cc4a5 109#if HAVE_LZ4
d89c8fdf
ZJS		 110 int r;
111
112 assert(src);
113 assert(src_size > 0);
114 assert(dst);
5d6f46b6 115 assert(dst_alloc_size > 0);
d89c8fdf
ZJS		 116 assert(dst_size);
117
118 /* Returns < 0 if we couldn't compress the data or the
119 * compressed result is longer than the original */
120
121 if (src_size < 9)
122 return -ENOBUFS;
e4e61fdb 123
691b90d4 124 r = LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
d89c8fdf
ZJS		 125 if (r <= 0)
126 return -ENOBUFS;
127
4094c4bf 128 unaligned_write_le64(dst, src_size);
d89c8fdf
ZJS		 129 *dst_size = r + 8;
130
131 return 0;
132#else
133 return -EPROTONOSUPPORT;
134#endif
135}
136
8653185a
LP		 137int compress_blob_zstd(
138 const void *src, uint64_t src_size,
139 void *dst, size_t dst_alloc_size, size_t *dst_size) {
140#if HAVE_ZSTD
141 size_t k;
142
143 assert(src);
144 assert(src_size > 0);
145 assert(dst);
146 assert(dst_alloc_size > 0);
147 assert(dst_size);
148
149 k = ZSTD_compress(dst, dst_alloc_size, src, src_size, 0);
150 if (ZSTD_isError(k))
151 return zstd_ret_to_errno(k);
152
153 *dst_size = k;
154 return 0;
155#else
156 return -EPROTONOSUPPORT;
157#endif
158}
159
d89c8fdf 160int decompress_blob_xz(const void *src, uint64_t src_size,
fa1c4b51 161 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
d89c8fdf 162
349cc4a5 163#if HAVE_XZ
5e592c66 164 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
e4e61fdb 165 lzma_ret ret;
fa1c4b51 166 size_t space;
e4e61fdb
LP		 167
168 assert(src);
169 assert(src_size > 0);
170 assert(dst);
171 assert(dst_alloc_size);
172 assert(dst_size);
173 assert(*dst_alloc_size == 0 || *dst);
174
175 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
176 if (ret != LZMA_OK)
d89c8fdf 177 return -ENOMEM;
e4e61fdb 178
f5fbe71d 179 space = MIN(src_size * 2, dst_max ?: SIZE_MAX);
5e592c66 180 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
01c3322e 181 return -ENOMEM;
e4e61fdb
LP		 182
183 s.next_in = src;
184 s.avail_in = src_size;
185
186 s.next_out = *dst;
93b73b06 187 s.avail_out = space;
e4e61fdb
LP		 188
189 for (;;) {
fa1c4b51 190 size_t used;
e4e61fdb
LP		 191
192 ret = lzma_code(&s, LZMA_FINISH);
193
194 if (ret == LZMA_STREAM_END)
195 break;
d89c8fdf
ZJS		 196 else if (ret != LZMA_OK)
197 return -ENOMEM;
e4e61fdb 198
93b73b06
LP		 199 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
200 break;
d89c8fdf
ZJS		 201 else if (dst_max > 0 && space == dst_max)
202 return -ENOBUFS;
93b73b06 203
5e592c66 204 used = space - s.avail_out;
f5fbe71d 205 space = MIN(2 * space, dst_max ?: SIZE_MAX);
5e592c66 206 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
01c3322e 207 return -ENOMEM;
e4e61fdb 208
5e592c66 209 s.avail_out = space - used;
8e170d29 210 s.next_out = *(uint8_t**)dst + used;
e4e61fdb
LP		 211 }
212
93b73b06 213 *dst_size = space - s.avail_out;
d89c8fdf
ZJS		 214 return 0;
215#else
216 return -EPROTONOSUPPORT;
217#endif
e4e61fdb
LP		 218}
219
d89c8fdf 220int decompress_blob_lz4(const void *src, uint64_t src_size,
fa1c4b51 221 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
d89c8fdf 222
349cc4a5 223#if HAVE_LZ4
d89c8fdf 224 char* out;
fa1c4b51 225 int r, size; /* LZ4 uses int for size */
e4e61fdb 226
d89c8fdf
ZJS		 227 assert(src);
228 assert(src_size > 0);
229 assert(dst);
230 assert(dst_alloc_size);
231 assert(dst_size);
232 assert(*dst_alloc_size == 0 || *dst);
233
234 if (src_size <= 8)
235 return -EBADMSG;
236
4094c4bf
LP		 237 size = unaligned_read_le64(src);
238 if (size < 0 || (unsigned) size != unaligned_read_le64(src))
fa1c4b51
ZJS		 239 return -EFBIG;
240 if ((size_t) size > *dst_alloc_size) {
d89c8fdf
ZJS		 241 out = realloc(*dst, size);
242 if (!out)
243 return -ENOMEM;
244 *dst = out;
245 *dst_alloc_size = size;
246 } else
247 out = *dst;
248
8e170d29 249 r = LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size);
fa1c4b51 250 if (r < 0 || r != size)
d89c8fdf
ZJS		 251 return -EBADMSG;
252
253 *dst_size = size;
254 return 0;
255#else
256 return -EPROTONOSUPPORT;
257#endif
258}
259
8653185a
LP		 260int decompress_blob_zstd(
261 const void *src, uint64_t src_size,
a2415327 262 void **dst, size_t *dst_alloc_size, size_t *dst_size, size_t dst_max) {
8653185a
LP		 263
264#if HAVE_ZSTD
a2415327 265 uint64_t size;
8653185a
LP		 266
267 assert(src);
268 assert(src_size > 0);
269 assert(dst);
270 assert(dst_alloc_size);
271 assert(dst_size);
272 assert(*dst_alloc_size == 0 || *dst);
273
a2415327
ZJS		 274 size = ZSTD_getFrameContentSize(src, src_size);
275 if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
276 return -EBADMSG;
8653185a 277
a2415327
ZJS		 278 if (dst_max > 0 && size > dst_max)
279 size = dst_max;
280 if (size > SIZE_MAX)
281 return -E2BIG;
8653185a 282
a2415327
ZJS		 283 if (!(greedy_realloc(dst, dst_alloc_size, MAX(ZSTD_DStreamOutSize(), size), 1)))
284 return -ENOMEM;
8653185a 285
a2415327
ZJS		 286 _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx();
287 if (!dctx)
288 return -ENOMEM;
8653185a 289
a2415327
ZJS		 290 ZSTD_inBuffer input = {
291 .src = src,
292 .size = src_size,
293 };
294 ZSTD_outBuffer output = {
295 .dst = *dst,
296 .size = *dst_alloc_size,
297 };
8653185a 298
a2415327
ZJS		 299 size_t k = ZSTD_decompressStream(dctx, &output, &input);
300 if (ZSTD_isError(k)) {
301 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k));
302 return zstd_ret_to_errno(k);
8653185a 303 }
a2415327
ZJS		 304 assert(output.pos >= size);
305
306 *dst_size = size;
307 return 0;
8653185a
LP		 308#else
309 return -EPROTONOSUPPORT;
310#endif
311}
312
313int decompress_blob(
314 int compression,
315 const void *src, uint64_t src_size,
316 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
317
d89c8fdf 318 if (compression == OBJECT_COMPRESSED_XZ)
8653185a
LP		 319 return decompress_blob_xz(
320 src, src_size,
321 dst, dst_alloc_size, dst_size, dst_max);
d89c8fdf 322 else if (compression == OBJECT_COMPRESSED_LZ4)
8653185a
LP		 323 return decompress_blob_lz4(
324 src, src_size,
325 dst, dst_alloc_size, dst_size, dst_max);
326 else if (compression == OBJECT_COMPRESSED_ZSTD)
327 return decompress_blob_zstd(
328 src, src_size,
329 dst, dst_alloc_size, dst_size, dst_max);
d89c8fdf 330 else
b4a11ca3 331 return -EPROTONOSUPPORT;
d89c8fdf
ZJS		 332}
333
d89c8fdf 334int decompress_startswith_xz(const void *src, uint64_t src_size,
fa1c4b51
ZJS		 335 void **buffer, size_t *buffer_size,
336 const void *prefix, size_t prefix_len,
d89c8fdf
ZJS		 337 uint8_t extra) {
338
349cc4a5 339#if HAVE_XZ
5e592c66 340 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
e4e61fdb 341 lzma_ret ret;
e4e61fdb 342
d89c8fdf 343 /* Checks whether the decompressed blob starts with the
e4e61fdb
LP		 344 * mentioned prefix. The byte extra needs to follow the
345 * prefix */
346
347 assert(src);
348 assert(src_size > 0);
349 assert(buffer);
350 assert(buffer_size);
351 assert(prefix);
352 assert(*buffer_size == 0 || *buffer);
353
354 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
355 if (ret != LZMA_OK)
d89c8fdf 356 return -EBADMSG;
e4e61fdb 357
d89c8fdf
ZJS		 358 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
359 return -ENOMEM;
e4e61fdb
LP		 360
361 s.next_in = src;
362 s.avail_in = src_size;
363
364 s.next_out = *buffer;
365 s.avail_out = *buffer_size;
366
367 for (;;) {
e4e61fdb
LP		 368 ret = lzma_code(&s, LZMA_FINISH);
369
4c701096 370 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
d89c8fdf 371 return -EBADMSG;
e4e61fdb 372
5e592c66
ZJS		 373 if (*buffer_size - s.avail_out >= prefix_len + 1)
374 return memcmp(*buffer, prefix, prefix_len) == 0 &&
375 ((const uint8_t*) *buffer)[prefix_len] == extra;
e4e61fdb
LP		 376
377 if (ret == LZMA_STREAM_END)
d89c8fdf 378 return 0;
e4e61fdb 379
e4e61fdb
LP		 380 s.avail_out += *buffer_size;
381
5e592c66 382 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
d89c8fdf 383 return -ENOMEM;
e4e61fdb 384
8e170d29 385 s.next_out = *(uint8_t**)buffer + *buffer_size - s.avail_out;
5e592c66 386 }
d89c8fdf
ZJS		 387
388#else
389 return -EPROTONOSUPPORT;
390#endif
391}
392
393int decompress_startswith_lz4(const void *src, uint64_t src_size,
fa1c4b51
ZJS		 394 void **buffer, size_t *buffer_size,
395 const void *prefix, size_t prefix_len,
d89c8fdf 396 uint8_t extra) {
349cc4a5 397#if HAVE_LZ4
d89c8fdf
ZJS		 398 /* Checks whether the decompressed blob starts with the
399 * mentioned prefix. The byte extra needs to follow the
400 * prefix */
401
402 int r;
403
404 assert(src);
405 assert(src_size > 0);
406 assert(buffer);
407 assert(buffer_size);
408 assert(prefix);
409 assert(*buffer_size == 0 || *buffer);
410
411 if (src_size <= 8)
412 return -EBADMSG;
413
414 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
415 return -ENOMEM;
416
8e170d29 417 r = LZ4_decompress_safe_partial((char*)src + 8, *buffer, src_size - 8,
d89c8fdf 418 prefix_len + 1, *buffer_size);
e41ef6fd
ZJS		 419 /* One lz4 < 1.8.3, we might get "failure" (r < 0), or "success" where
420 * just a part of the buffer is decompressed. But if we get a smaller
421 * amount of bytes than requested, we don't know whether there isn't enough
422 * data to fill the requested size or whether we just got a partial answer.
423 */
424 if (r < 0 || (size_t) r < prefix_len + 1) {
425 size_t size;
426
427 if (LZ4_versionNumber() >= 10803)
428 /* We trust that the newer lz4 decompresses the number of bytes we
429 * requested if available in the compressed string. */
430 return 0;
431
432 if (r > 0)
433 /* Compare what we have first, in case of mismatch we can
434 * shortcut the full comparison. */
435 if (memcmp(*buffer, prefix, r) != 0)
436 return 0;
437
438 /* Before version 1.8.3, lz4 always tries to decode full a "sequence",
439 * so in pathological cases might need to decompress the full field. */
1f4b467d
ZJS		 440 r = decompress_blob_lz4(src, src_size, buffer, buffer_size, &size, 0);
441 if (r < 0)
442 return r;
d89c8fdf 443
e41ef6fd
ZJS		 444 if (size < prefix_len + 1)
445 return 0;
446 }
d89c8fdf 447
e41ef6fd
ZJS		 448 return memcmp(*buffer, prefix, prefix_len) == 0 &&
449 ((const uint8_t*) *buffer)[prefix_len] == extra;
d89c8fdf
ZJS		 450#else
451 return -EPROTONOSUPPORT;
452#endif
e4e61fdb 453}
355b59e2 454
8653185a
LP		 455int decompress_startswith_zstd(
456 const void *src, uint64_t src_size,
457 void **buffer, size_t *buffer_size,
458 const void *prefix, size_t prefix_len,
459 uint8_t extra) {
460#if HAVE_ZSTD
8653185a
LP		 461 assert(src);
462 assert(src_size > 0);
463 assert(buffer);
464 assert(buffer_size);
465 assert(prefix);
466 assert(*buffer_size == 0 || *buffer);
467
e4a321fc
ZJS		 468 uint64_t size = ZSTD_getFrameContentSize(src, src_size);
469 if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
470 return -EBADMSG;
471
472 if (size < prefix_len + 1)
473 return 0; /* Decompressed text too short to match the prefix and extra */
474
475 _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx();
8653185a
LP		 476 if (!dctx)
477 return -ENOMEM;
478
479 if (!(greedy_realloc(buffer, buffer_size, MAX(ZSTD_DStreamOutSize(), prefix_len + 1), 1)))
480 return -ENOMEM;
481
482 ZSTD_inBuffer input = {
483 .src = src,
484 .size = src_size,
485 };
486 ZSTD_outBuffer output = {
487 .dst = *buffer,
488 .size = *buffer_size,
489 };
e4a321fc 490 size_t k;
8653185a 491
e4a321fc
ZJS		 492 k = ZSTD_decompressStream(dctx, &output, &input);
493 if (ZSTD_isError(k)) {
494 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k));
495 return zstd_ret_to_errno(k);
8653185a 496 }
e4a321fc
ZJS		 497 assert(output.pos >= prefix_len + 1);
498
499 return memcmp(*buffer, prefix, prefix_len) == 0 &&
500 ((const uint8_t*) *buffer)[prefix_len] == extra;
8653185a
LP		 501#else
502 return -EPROTONOSUPPORT;
503#endif
504}
505
506int decompress_startswith(
507 int compression,
508 const void *src, uint64_t src_size,
509 void **buffer, size_t *buffer_size,
510 const void *prefix, size_t prefix_len,
511 uint8_t extra) {
512
d89c8fdf 513 if (compression == OBJECT_COMPRESSED_XZ)
8653185a
LP		 514 return decompress_startswith_xz(
515 src, src_size,
516 buffer, buffer_size,
517 prefix, prefix_len,
518 extra);
519
d89c8fdf 520 else if (compression == OBJECT_COMPRESSED_LZ4)
8653185a
LP		 521 return decompress_startswith_lz4(
522 src, src_size,
523 buffer, buffer_size,
524 prefix, prefix_len,
525 extra);
526 else if (compression == OBJECT_COMPRESSED_ZSTD)
527 return decompress_startswith_zstd(
528 src, src_size,
529 buffer, buffer_size,
530 prefix, prefix_len,
531 extra);
d89c8fdf
ZJS		 532 else
533 return -EBADMSG;
534}
535
59f448cf 536int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
349cc4a5 537#if HAVE_XZ
355b59e2
ZJS		 538 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
539 lzma_ret ret;
355b59e2
ZJS		 540 uint8_t buf[BUFSIZ], out[BUFSIZ];
541 lzma_action action = LZMA_RUN;
542
543 assert(fdf >= 0);
544 assert(fdt >= 0);
545
d89c8fdf 546 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
d7a0f1f4
FS		 547 if (ret != LZMA_OK)
548 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
549 "Failed to initialize XZ encoder: code %u",
550 ret);
355b59e2
ZJS		 551
552 for (;;) {
553 if (s.avail_in == 0 && action == LZMA_RUN) {
554 size_t m = sizeof(buf);
555 ssize_t n;
556
f5fbe71d 557 if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes)
59f448cf 558 m = (size_t) max_bytes;
355b59e2
ZJS		 559
560 n = read(fdf, buf, m);
561 if (n < 0)
562 return -errno;
563 if (n == 0)
564 action = LZMA_FINISH;
565 else {
566 s.next_in = buf;
567 s.avail_in = n;
568
f5fbe71d 569 if (max_bytes != UINT64_MAX) {
59f448cf 570 assert(max_bytes >= (uint64_t) n);
355b59e2
ZJS		 571 max_bytes -= n;
572 }
573 }
574 }
575
576 if (s.avail_out == 0) {
577 s.next_out = out;
578 s.avail_out = sizeof(out);
579 }
580
581 ret = lzma_code(&s, action);
d7a0f1f4
FS		 582 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
583 return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
584 "Compression failed: code %u",
585 ret);
355b59e2
ZJS		 586
587 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
588 ssize_t n, k;
589
590 n = sizeof(out) - s.avail_out;
591
355b59e2
ZJS		 592 k = loop_write(fdt, out, n, false);
593 if (k < 0)
594 return k;
355b59e2
ZJS		 595
596 if (ret == LZMA_STREAM_END) {
fa1c4b51 597 log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
355b59e2
ZJS		 598 s.total_in, s.total_out,
599 (double) s.total_out / s.total_in * 100);
600
601 return 0;
602 }
603 }
604 }
3b1a55e1
ZJS		 605#else
606 return -EPROTONOSUPPORT;
607#endif
355b59e2
ZJS		 608}
609
4b5bc539 610#define LZ4_BUFSIZE (512*1024u)
d89c8fdf 611
59f448cf 612int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
d89c8fdf 613
349cc4a5 614#if HAVE_LZ4
4b5bc539
ZJS		 615 LZ4F_errorCode_t c;
616 _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
834bab01
LB		 617 _cleanup_free_ void *in_buff = NULL;
618 _cleanup_free_ char *out_buff = NULL;
619 size_t out_allocsize, n, total_in = 0, total_out, offset = 0, frame_size;
4b5bc539 620 int r;
4b5bc539
ZJS		 621 static const LZ4F_preferences_t preferences = {
622 .frameInfo.blockSizeID = 5,
623 };
d89c8fdf 624
4b5bc539
ZJS		 625 c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
626 if (LZ4F_isError(c))
627 return -ENOMEM;
d89c8fdf 628
4b5bc539 629 frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
834bab01
LB		 630 out_allocsize = frame_size + 64*1024; /* add some space for header and trailer */
631 out_buff = malloc(out_allocsize);
632 if (!out_buff)
633 return -ENOMEM;
634
635 in_buff = malloc(LZ4_BUFSIZE);
636 if (!in_buff)
4b5bc539 637 return -ENOMEM;
d89c8fdf 638
834bab01 639 n = offset = total_out = LZ4F_compressBegin(ctx, out_buff, out_allocsize, &preferences);
4b5bc539
ZJS		 640 if (LZ4F_isError(n))
641 return -EINVAL;
d89c8fdf 642
834bab01 643 log_debug("Buffer size is %zu bytes, header size %zu bytes.", out_allocsize, n);
d89c8fdf 644
834bab01 645 for (;;) {
4b5bc539 646 ssize_t k;
d89c8fdf 647
834bab01
LB		 648 k = loop_read(fdf, in_buff, LZ4_BUFSIZE, true);
649 if (k < 0)
650 return k;
651 if (k == 0)
652 break;
653 n = LZ4F_compressUpdate(ctx, out_buff + offset, out_allocsize - offset,
654 in_buff, k, NULL);
655 if (LZ4F_isError(n))
656 return -ENOTRECOVERABLE;
d89c8fdf 657
4b5bc539
ZJS		 658 total_in += k;
659 offset += n;
660 total_out += n;
d89c8fdf 661
834bab01
LB		 662 if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes)
663 return log_debug_errno(SYNTHETIC_ERRNO(EFBIG),
664 "Compressed stream longer than %" PRIu64 " bytes", max_bytes);
d89c8fdf 665
834bab01
LB		 666 if (out_allocsize - offset < frame_size + 4) {
667 k = loop_write(fdt, out_buff, offset, false);
668 if (k < 0)
669 return k;
4b5bc539
ZJS		 670 offset = 0;
671 }
672 }
d89c8fdf 673
834bab01
LB		 674 n = LZ4F_compressEnd(ctx, out_buff + offset, out_allocsize - offset, NULL);
675 if (LZ4F_isError(n))
676 return -ENOTRECOVERABLE;
d89c8fdf 677
4b5bc539
ZJS		 678 offset += n;
679 total_out += n;
834bab01 680 r = loop_write(fdt, out_buff, offset, false);
4b5bc539 681 if (r < 0)
834bab01 682 return r;
d89c8fdf
ZJS		 683
684 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
685 total_in, total_out,
686 (double) total_out / total_in * 100);
834bab01
LB		 687
688 return 0;
d89c8fdf
ZJS		 689#else
690 return -EPROTONOSUPPORT;
691#endif
692}
693
59f448cf 694int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
d89c8fdf 695
349cc4a5 696#if HAVE_XZ
355b59e2
ZJS		 697 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
698 lzma_ret ret;
699
700 uint8_t buf[BUFSIZ], out[BUFSIZ];
701 lzma_action action = LZMA_RUN;
702
703 assert(fdf >= 0);
704 assert(fdt >= 0);
705
706 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
d7a0f1f4
FS		 707 if (ret != LZMA_OK)
708 return log_debug_errno(SYNTHETIC_ERRNO(ENOMEM),
709 "Failed to initialize XZ decoder: code %u",
710 ret);
355b59e2
ZJS		 711
712 for (;;) {
713 if (s.avail_in == 0 && action == LZMA_RUN) {
714 ssize_t n;
715
716 n = read(fdf, buf, sizeof(buf));
717 if (n < 0)
718 return -errno;
719 if (n == 0)
720 action = LZMA_FINISH;
721 else {
722 s.next_in = buf;
723 s.avail_in = n;
724 }
725 }
726
727 if (s.avail_out == 0) {
728 s.next_out = out;
729 s.avail_out = sizeof(out);
730 }
731
732 ret = lzma_code(&s, action);
d7a0f1f4
FS		 733 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
734 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
735 "Decompression failed: code %u",
736 ret);
355b59e2
ZJS		 737
738 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
739 ssize_t n, k;
740
741 n = sizeof(out) - s.avail_out;
742
f5fbe71d 743 if (max_bytes != UINT64_MAX) {
59f448cf 744 if (max_bytes < (uint64_t) n)
d89c8fdf 745 return -EFBIG;
355b59e2
ZJS		 746
747 max_bytes -= n;
748 }
749
355b59e2
ZJS		 750 k = loop_write(fdt, out, n, false);
751 if (k < 0)
752 return k;
355b59e2
ZJS		 753
754 if (ret == LZMA_STREAM_END) {
fa1c4b51 755 log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
355b59e2
ZJS		 756 s.total_in, s.total_out,
757 (double) s.total_out / s.total_in * 100);
758
759 return 0;
760 }
761 }
762 }
d89c8fdf 763#else
d7a0f1f4
FS		 764 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
765 "Cannot decompress file. Compiled without XZ support.");
d89c8fdf
ZJS		 766#endif
767}
768
8e64dd1e 769int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
349cc4a5 770#if HAVE_LZ4
4b5bc539
ZJS		 771 size_t c;
772 _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
773 _cleanup_free_ char *buf = NULL;
774 char *src;
775 struct stat st;
776 int r = 0;
777 size_t total_in = 0, total_out = 0;
778
779 c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
780 if (LZ4F_isError(c))
781 return -ENOMEM;
782
783 if (fstat(in, &st) < 0)
5146f9f0 784 return log_debug_errno(errno, "fstat() failed: %m");
4b5bc539
ZJS		 785
786 buf = malloc(LZ4_BUFSIZE);
787 if (!buf)
788 return -ENOMEM;
789
790 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in, 0);
e0685172 791 if (src == MAP_FAILED)
4b5bc539
ZJS		 792 return -errno;
793
794 while (total_in < (size_t) st.st_size) {
795 size_t produced = LZ4_BUFSIZE;
796 size_t used = st.st_size - total_in;
797
798 c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
799 if (LZ4F_isError(c)) {
800 r = -EBADMSG;
801 goto cleanup;
802 }
803
804 total_in += used;
805 total_out += produced;
806
f5fbe71d 807 if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes) {
82e24b00 808 log_debug("Decompressed stream longer than %"PRIu64" bytes", max_bytes);
5146f9f0 809 r = -EFBIG;
4b5bc539
ZJS		 810 goto cleanup;
811 }
812
813 r = loop_write(out, buf, produced, false);
814 if (r < 0)
815 goto cleanup;
816 }
817
818 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
819 total_in, total_out,
25048348 820 total_in > 0 ? (double) total_out / total_in * 100 : 0.0);
4b5bc539
ZJS		 821 cleanup:
822 munmap(src, st.st_size);
823 return r;
d89c8fdf 824#else
d7a0f1f4
FS		 825 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
826 "Cannot decompress file. Compiled without LZ4 support.");
d89c8fdf
ZJS		 827#endif
828}
829
ef5924aa
NL		 830int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
831#if HAVE_ZSTD
832 _cleanup_(ZSTD_freeCCtxp) ZSTD_CCtx *cctx = NULL;
833 _cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
834 size_t in_allocsize, out_allocsize;
835 size_t z;
836 uint64_t left = max_bytes, in_bytes = 0;
ef5924aa
NL		 837
838 assert(fdf >= 0);
839 assert(fdt >= 0);
840
841 /* Create the context and buffers */
842 in_allocsize = ZSTD_CStreamInSize();
843 out_allocsize = ZSTD_CStreamOutSize();
844 in_buff = malloc(in_allocsize);
845 out_buff = malloc(out_allocsize);
846 cctx = ZSTD_createCCtx();
847 if (!cctx || !out_buff || !in_buff)
848 return -ENOMEM;
849
ef5924aa
NL		 850 z = ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1);
851 if (ZSTD_isError(z))
852 log_debug("Failed to enable ZSTD checksum, ignoring: %s", ZSTD_getErrorName(z));
853
854 /* This loop read from the input file, compresses that entire chunk,
855 * and writes all output produced to the output file.
856 */
857 for (;;) {
858 bool is_last_chunk;
859 ZSTD_inBuffer input = {
860 .src = in_buff,
861 .size = 0,
862 .pos = 0
863 };
864 ssize_t red;
865
866 red = loop_read(fdf, in_buff, in_allocsize, true);
867 if (red < 0)
868 return red;
869 is_last_chunk = red == 0;
870
871 in_bytes += (size_t) red;
872 input.size = (size_t) red;
873
874 for (bool finished = false; !finished;) {
875 ZSTD_outBuffer output = {
876 .dst = out_buff,
877 .size = out_allocsize,
878 .pos = 0
879 };
880 size_t remaining;
881 ssize_t wrote;
882
883 /* Compress into the output buffer and write all of the
884 * output to the file so we can reuse the buffer next
885 * iteration.
886 */
887 remaining = ZSTD_compressStream2(
888 cctx, &output, &input,
889 is_last_chunk ? ZSTD_e_end : ZSTD_e_continue);
890
891 if (ZSTD_isError(remaining)) {
892 log_debug("ZSTD encoder failed: %s", ZSTD_getErrorName(remaining));
893 return zstd_ret_to_errno(remaining);
894 }
895
896 if (left < output.pos)
897 return -EFBIG;
898
899 wrote = loop_write(fdt, output.dst, output.pos, 1);
900 if (wrote < 0)
901 return wrote;
902
903 left -= output.pos;
904
905 /* If we're on the last chunk we're finished when zstd
906 * returns 0, which means its consumed all the input AND
907 * finished the frame. Otherwise, we're finished when
908 * we've consumed all the input.
909 */
910 finished = is_last_chunk ? (remaining == 0) : (input.pos == input.size);
911 }
912
913 /* zstd only returns 0 when the input is completely consumed */
914 assert(input.pos == input.size);
915 if (is_last_chunk)
916 break;
917 }
918
89d36ce8
YW		 919 if (in_bytes > 0)
920 log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
921 in_bytes, max_bytes - left, (double) (max_bytes - left) / in_bytes * 100);
922 else
923 log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes)",
924 in_bytes, max_bytes - left);
ef5924aa
NL		 925
926 return 0;
927#else
928 return -EPROTONOSUPPORT;
929#endif
930}
931
932int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
933#if HAVE_ZSTD
934 _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
935 _cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
936 size_t in_allocsize, out_allocsize;
937 size_t last_result = 0;
938 uint64_t left = max_bytes, in_bytes = 0;
939
940 assert(fdf >= 0);
941 assert(fdt >= 0);
942
943 /* Create the context and buffers */
944 in_allocsize = ZSTD_DStreamInSize();
945 out_allocsize = ZSTD_DStreamOutSize();
946 in_buff = malloc(in_allocsize);
947 out_buff = malloc(out_allocsize);
948 dctx = ZSTD_createDCtx();
949 if (!dctx || !out_buff || !in_buff)
950 return -ENOMEM;
951
952 /* This loop assumes that the input file is one or more concatenated
953 * zstd streams. This example won't work if there is trailing non-zstd
954 * data at the end, but streaming decompression in general handles this
955 * case. ZSTD_decompressStream() returns 0 exactly when the frame is
956 * completed, and doesn't consume input after the frame.
957 */
958 for (;;) {
959 bool has_error = false;
960 ZSTD_inBuffer input = {
961 .src = in_buff,
962 .size = 0,
963 .pos = 0
964 };
965 ssize_t red;
966
967 red = loop_read(fdf, in_buff, in_allocsize, true);
968 if (red < 0)
969 return red;
970 if (red == 0)
971 break;
972
973 in_bytes += (size_t) red;
974 input.size = (size_t) red;
975 input.pos = 0;
976
977 /* Given a valid frame, zstd won't consume the last byte of the
978 * frame until it has flushed all of the decompressed data of
979 * the frame. So input.pos < input.size means frame is not done
980 * or there is still output available.
981 */
982 while (input.pos < input.size) {
983 ZSTD_outBuffer output = {
984 .dst = out_buff,
985 .size = out_allocsize,
986 .pos = 0
987 };
988 ssize_t wrote;
989 /* The return code is zero if the frame is complete, but
990 * there may be multiple frames concatenated together.
991 * Zstd will automatically reset the context when a
992 * frame is complete. Still, calling ZSTD_DCtx_reset()
993 * can be useful to reset the context to a clean state,
994 * for instance if the last decompression call returned
995 * an error.
996 */
997 last_result = ZSTD_decompressStream(dctx, &output, &input);
998 if (ZSTD_isError(last_result)) {
999 has_error = true;
1000 break;
1001 }
1002
1003 if (left < output.pos)
1004 return -EFBIG;
1005
1006 wrote = loop_write(fdt, output.dst, output.pos, 1);
1007 if (wrote < 0)
1008 return wrote;
1009
1010 left -= output.pos;
1011 }
1012 if (has_error)
1013 break;
1014 }
1015
1016 if (in_bytes == 0)
1017 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD decoder failed: no data read");
1018
1019 if (last_result != 0) {
1020 /* The last return value from ZSTD_decompressStream did not end
1021 * on a frame, but we reached the end of the file! We assume
1022 * this is an error, and the input was truncated.
1023 */
1024 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(last_result));
1025 return zstd_ret_to_errno(last_result);
1026 }
1027
1028 log_debug(
1029 "ZSTD decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
1030 in_bytes,
1031 max_bytes - left,
1032 (double) (max_bytes - left) / in_bytes * 100);
1033 return 0;
1034#else
d7a0f1f4
FS		 1035 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
1036 "Cannot decompress file. Compiled without ZSTD support.");
ef5924aa
NL		 1037#endif
1038}
1039
59f448cf 1040int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
d89c8fdf
ZJS		 1041
1042 if (endswith(filename, ".lz4"))
1043 return decompress_stream_lz4(fdf, fdt, max_bytes);
1044 else if (endswith(filename, ".xz"))
1045 return decompress_stream_xz(fdf, fdt, max_bytes);
ef5924aa
NL		 1046 else if (endswith(filename, ".zst"))
1047 return decompress_stream_zstd(fdf, fdt, max_bytes);
d89c8fdf
ZJS		 1048 else
1049 return -EPROTONOSUPPORT;
355b59e2 1050}
Unnamed repository; edit this file 'description' to name the repository.