]> git.ipfire.org Git - thirdparty/systemd.git/blame - src/libsystemd/sd-journal/compress.c
projects / thirdparty / systemd.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tree-wide: use UINT64_MAX or friends
[thirdparty/systemd.git] / src / libsystemd / sd-journal / compress.c
CommitLineData
db9ecf05 1/* SPDX-License-Identifier: LGPL-2.1-or-later */
e4e61fdb 2
82e24b00 3#include <inttypes.h>
e4e61fdb 4#include <stdlib.h>
4b5bc539 5#include <sys/mman.h>
ca78ad1d
ZJS		 6#include <sys/types.h>
7#include <sys/stat.h>
07630cea 8#include <unistd.h>
d89c8fdf 9
349cc4a5 10#if HAVE_XZ
3ffd4af2 11#include <lzma.h>
d89c8fdf
ZJS		 12#endif
13
349cc4a5 14#if HAVE_LZ4
3ffd4af2
LP		 15#include <lz4.h>
16#include <lz4frame.h>
d89c8fdf 17#endif
e4e61fdb 18
ef5924aa
NL		 19#if HAVE_ZSTD
20#include <zstd.h>
21#include <zstd_errors.h>
22#endif
23
b5efdb8a 24#include "alloc-util.h"
e4e61fdb 25#include "compress.h"
3ffd4af2 26#include "fd-util.h"
c004493c 27#include "io-util.h"
07630cea 28#include "journal-def.h"
355b59e2 29#include "macro.h"
d89c8fdf 30#include "sparse-endian.h"
8b43440b 31#include "string-table.h"
07630cea 32#include "string-util.h"
4094c4bf 33#include "unaligned.h"
07630cea 34#include "util.h"
d89c8fdf 35
349cc4a5 36#if HAVE_LZ4
fd421c4a
ZJS		 37DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_compressionContext_t, LZ4F_freeCompressionContext, NULL);
38DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext, NULL);
4b5bc539
ZJS		 39#endif
40
ef5924aa 41#if HAVE_ZSTD
fd421c4a
ZJS		 42DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_CCtx*, ZSTD_freeCCtx, NULL);
43DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_DCtx*, ZSTD_freeDCtx, NULL);
ef5924aa
NL		 44
45static int zstd_ret_to_errno(size_t ret) {
46 switch (ZSTD_getErrorCode(ret)) {
47 case ZSTD_error_dstSize_tooSmall:
48 return -ENOBUFS;
49 case ZSTD_error_memory_allocation:
50 return -ENOMEM;
51 default:
52 return -EBADMSG;
53 }
54}
55#endif
56
d89c8fdf
ZJS		 57#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
58
59static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
e9ece6a0
LP		 60 [OBJECT_COMPRESSED_XZ] = "XZ",
61 [OBJECT_COMPRESSED_LZ4] = "LZ4",
8653185a 62 [OBJECT_COMPRESSED_ZSTD] = "ZSTD",
e9ece6a0
LP		 63 /* If we add too many more entries here, it's going to grow quite large (and be mostly sparse), since
64 * the array key is actually a bitmask, not a plain enum */
d89c8fdf 65};
e4e61fdb 66
d89c8fdf
ZJS		 67DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
68
5d6f46b6
ZJS		 69int compress_blob_xz(const void *src, uint64_t src_size,
70 void *dst, size_t dst_alloc_size, size_t *dst_size) {
349cc4a5 71#if HAVE_XZ
1930eed2
JS		 72 static const lzma_options_lzma opt = {
73 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
0240c603
LP		 74 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
75 };
76 static const lzma_filter filters[] = {
77 { LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt },
78 { LZMA_VLI_UNKNOWN, NULL }
1930eed2 79 };
e4e61fdb 80 lzma_ret ret;
76cc0bf6 81 size_t out_pos = 0;
e4e61fdb
LP		 82
83 assert(src);
84 assert(src_size > 0);
85 assert(dst);
5d6f46b6 86 assert(dst_alloc_size > 0);
e4e61fdb
LP		 87 assert(dst_size);
88
d89c8fdf 89 /* Returns < 0 if we couldn't compress the data or the
e4e61fdb
LP		 90 * compressed result is longer than the original */
91
1930eed2
JS		 92 if (src_size < 80)
93 return -ENOBUFS;
94
95 ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
5d6f46b6 96 src, src_size, dst, &out_pos, dst_alloc_size);
e4e61fdb 97 if (ret != LZMA_OK)
d89c8fdf 98 return -ENOBUFS;
e4e61fdb 99
76cc0bf6 100 *dst_size = out_pos;
d89c8fdf
ZJS		 101 return 0;
102#else
103 return -EPROTONOSUPPORT;
104#endif
e4e61fdb
LP		 105}
106
5d6f46b6
ZJS		 107int compress_blob_lz4(const void *src, uint64_t src_size,
108 void *dst, size_t dst_alloc_size, size_t *dst_size) {
349cc4a5 109#if HAVE_LZ4
d89c8fdf
ZJS		 110 int r;
111
112 assert(src);
113 assert(src_size > 0);
114 assert(dst);
5d6f46b6 115 assert(dst_alloc_size > 0);
d89c8fdf
ZJS		 116 assert(dst_size);
117
118 /* Returns < 0 if we couldn't compress the data or the
119 * compressed result is longer than the original */
120
121 if (src_size < 9)
122 return -ENOBUFS;
e4e61fdb 123
691b90d4 124 r = LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
d89c8fdf
ZJS		 125 if (r <= 0)
126 return -ENOBUFS;
127
4094c4bf 128 unaligned_write_le64(dst, src_size);
d89c8fdf
ZJS		 129 *dst_size = r + 8;
130
131 return 0;
132#else
133 return -EPROTONOSUPPORT;
134#endif
135}
136
8653185a
LP		 137int compress_blob_zstd(
138 const void *src, uint64_t src_size,
139 void *dst, size_t dst_alloc_size, size_t *dst_size) {
140#if HAVE_ZSTD
141 size_t k;
142
143 assert(src);
144 assert(src_size > 0);
145 assert(dst);
146 assert(dst_alloc_size > 0);
147 assert(dst_size);
148
149 k = ZSTD_compress(dst, dst_alloc_size, src, src_size, 0);
150 if (ZSTD_isError(k))
151 return zstd_ret_to_errno(k);
152
153 *dst_size = k;
154 return 0;
155#else
156 return -EPROTONOSUPPORT;
157#endif
158}
159
d89c8fdf 160int decompress_blob_xz(const void *src, uint64_t src_size,
fa1c4b51 161 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
d89c8fdf 162
349cc4a5 163#if HAVE_XZ
5e592c66 164 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
e4e61fdb 165 lzma_ret ret;
fa1c4b51 166 size_t space;
e4e61fdb
LP		 167
168 assert(src);
169 assert(src_size > 0);
170 assert(dst);
171 assert(dst_alloc_size);
172 assert(dst_size);
173 assert(*dst_alloc_size == 0 || *dst);
174
175 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
176 if (ret != LZMA_OK)
d89c8fdf 177 return -ENOMEM;
e4e61fdb 178
f5fbe71d 179 space = MIN(src_size * 2, dst_max ?: SIZE_MAX);
5e592c66 180 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
01c3322e 181 return -ENOMEM;
e4e61fdb
LP		 182
183 s.next_in = src;
184 s.avail_in = src_size;
185
186 s.next_out = *dst;
93b73b06 187 s.avail_out = space;
e4e61fdb
LP		 188
189 for (;;) {
fa1c4b51 190 size_t used;
e4e61fdb
LP		 191
192 ret = lzma_code(&s, LZMA_FINISH);
193
194 if (ret == LZMA_STREAM_END)
195 break;
d89c8fdf
ZJS		 196 else if (ret != LZMA_OK)
197 return -ENOMEM;
e4e61fdb 198
93b73b06
LP		 199 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
200 break;
d89c8fdf
ZJS		 201 else if (dst_max > 0 && space == dst_max)
202 return -ENOBUFS;
93b73b06 203
5e592c66 204 used = space - s.avail_out;
f5fbe71d 205 space = MIN(2 * space, dst_max ?: SIZE_MAX);
5e592c66 206 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
01c3322e 207 return -ENOMEM;
e4e61fdb 208
5e592c66 209 s.avail_out = space - used;
8e170d29 210 s.next_out = *(uint8_t**)dst + used;
e4e61fdb
LP		 211 }
212
93b73b06 213 *dst_size = space - s.avail_out;
d89c8fdf
ZJS		 214 return 0;
215#else
216 return -EPROTONOSUPPORT;
217#endif
e4e61fdb
LP		 218}
219
d89c8fdf 220int decompress_blob_lz4(const void *src, uint64_t src_size,
fa1c4b51 221 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
d89c8fdf 222
349cc4a5 223#if HAVE_LZ4
d89c8fdf 224 char* out;
fa1c4b51 225 int r, size; /* LZ4 uses int for size */
e4e61fdb 226
d89c8fdf
ZJS		 227 assert(src);
228 assert(src_size > 0);
229 assert(dst);
230 assert(dst_alloc_size);
231 assert(dst_size);
232 assert(*dst_alloc_size == 0 || *dst);
233
234 if (src_size <= 8)
235 return -EBADMSG;
236
4094c4bf
LP		 237 size = unaligned_read_le64(src);
238 if (size < 0 || (unsigned) size != unaligned_read_le64(src))
fa1c4b51
ZJS		 239 return -EFBIG;
240 if ((size_t) size > *dst_alloc_size) {
d89c8fdf
ZJS		 241 out = realloc(*dst, size);
242 if (!out)
243 return -ENOMEM;
244 *dst = out;
245 *dst_alloc_size = size;
246 } else
247 out = *dst;
248
8e170d29 249 r = LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size);
fa1c4b51 250 if (r < 0 || r != size)
d89c8fdf
ZJS		 251 return -EBADMSG;
252
253 *dst_size = size;
254 return 0;
255#else
256 return -EPROTONOSUPPORT;
257#endif
258}
259
8653185a
LP		 260int decompress_blob_zstd(
261 const void *src, uint64_t src_size,
a2415327 262 void **dst, size_t *dst_alloc_size, size_t *dst_size, size_t dst_max) {
8653185a
LP		 263
264#if HAVE_ZSTD
a2415327 265 uint64_t size;
8653185a
LP		 266
267 assert(src);
268 assert(src_size > 0);
269 assert(dst);
270 assert(dst_alloc_size);
271 assert(dst_size);
272 assert(*dst_alloc_size == 0 || *dst);
273
a2415327
ZJS		 274 size = ZSTD_getFrameContentSize(src, src_size);
275 if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
276 return -EBADMSG;
8653185a 277
a2415327
ZJS		 278 if (dst_max > 0 && size > dst_max)
279 size = dst_max;
280 if (size > SIZE_MAX)
281 return -E2BIG;
8653185a 282
a2415327
ZJS		 283 if (!(greedy_realloc(dst, dst_alloc_size, MAX(ZSTD_DStreamOutSize(), size), 1)))
284 return -ENOMEM;
8653185a 285
a2415327
ZJS		 286 _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx();
287 if (!dctx)
288 return -ENOMEM;
8653185a 289
a2415327
ZJS		 290 ZSTD_inBuffer input = {
291 .src = src,
292 .size = src_size,
293 };
294 ZSTD_outBuffer output = {
295 .dst = *dst,
296 .size = *dst_alloc_size,
297 };
8653185a 298
a2415327
ZJS		 299 size_t k = ZSTD_decompressStream(dctx, &output, &input);
300 if (ZSTD_isError(k)) {
301 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k));
302 return zstd_ret_to_errno(k);
8653185a 303 }
a2415327
ZJS		 304 assert(output.pos >= size);
305
306 *dst_size = size;
307 return 0;
8653185a
LP		 308#else
309 return -EPROTONOSUPPORT;
310#endif
311}
312
313int decompress_blob(
314 int compression,
315 const void *src, uint64_t src_size,
316 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
317
d89c8fdf 318 if (compression == OBJECT_COMPRESSED_XZ)
8653185a
LP		 319 return decompress_blob_xz(
320 src, src_size,
321 dst, dst_alloc_size, dst_size, dst_max);
d89c8fdf 322 else if (compression == OBJECT_COMPRESSED_LZ4)
8653185a
LP		 323 return decompress_blob_lz4(
324 src, src_size,
325 dst, dst_alloc_size, dst_size, dst_max);
326 else if (compression == OBJECT_COMPRESSED_ZSTD)
327 return decompress_blob_zstd(
328 src, src_size,
329 dst, dst_alloc_size, dst_size, dst_max);
d89c8fdf 330 else
b4a11ca3 331 return -EPROTONOSUPPORT;
d89c8fdf
ZJS		 332}
333
d89c8fdf 334int decompress_startswith_xz(const void *src, uint64_t src_size,
fa1c4b51
ZJS		 335 void **buffer, size_t *buffer_size,
336 const void *prefix, size_t prefix_len,
d89c8fdf
ZJS		 337 uint8_t extra) {
338
349cc4a5 339#if HAVE_XZ
5e592c66 340 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
e4e61fdb 341 lzma_ret ret;
e4e61fdb 342
d89c8fdf 343 /* Checks whether the decompressed blob starts with the
e4e61fdb
LP		 344 * mentioned prefix. The byte extra needs to follow the
345 * prefix */
346
347 assert(src);
348 assert(src_size > 0);
349 assert(buffer);
350 assert(buffer_size);
351 assert(prefix);
352 assert(*buffer_size == 0 || *buffer);
353
354 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
355 if (ret != LZMA_OK)
d89c8fdf 356 return -EBADMSG;
e4e61fdb 357
d89c8fdf
ZJS		 358 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
359 return -ENOMEM;
e4e61fdb
LP		 360
361 s.next_in = src;
362 s.avail_in = src_size;
363
364 s.next_out = *buffer;
365 s.avail_out = *buffer_size;
366
367 for (;;) {
e4e61fdb
LP		 368 ret = lzma_code(&s, LZMA_FINISH);
369
4c701096 370 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
d89c8fdf 371 return -EBADMSG;
e4e61fdb 372
5e592c66
ZJS		 373 if (*buffer_size - s.avail_out >= prefix_len + 1)
374 return memcmp(*buffer, prefix, prefix_len) == 0 &&
375 ((const uint8_t*) *buffer)[prefix_len] == extra;
e4e61fdb
LP		 376
377 if (ret == LZMA_STREAM_END)
d89c8fdf 378 return 0;
e4e61fdb 379
e4e61fdb
LP		 380 s.avail_out += *buffer_size;
381
5e592c66 382 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
d89c8fdf 383 return -ENOMEM;
e4e61fdb 384
8e170d29 385 s.next_out = *(uint8_t**)buffer + *buffer_size - s.avail_out;
5e592c66 386 }
d89c8fdf
ZJS		 387
388#else
389 return -EPROTONOSUPPORT;
390#endif
391}
392
393int decompress_startswith_lz4(const void *src, uint64_t src_size,
fa1c4b51
ZJS		 394 void **buffer, size_t *buffer_size,
395 const void *prefix, size_t prefix_len,
d89c8fdf 396 uint8_t extra) {
349cc4a5 397#if HAVE_LZ4
d89c8fdf
ZJS		 398 /* Checks whether the decompressed blob starts with the
399 * mentioned prefix. The byte extra needs to follow the
400 * prefix */
401
402 int r;
403
404 assert(src);
405 assert(src_size > 0);
406 assert(buffer);
407 assert(buffer_size);
408 assert(prefix);
409 assert(*buffer_size == 0 || *buffer);
410
411 if (src_size <= 8)
412 return -EBADMSG;
413
414 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
415 return -ENOMEM;
416
8e170d29 417 r = LZ4_decompress_safe_partial((char*)src + 8, *buffer, src_size - 8,
d89c8fdf 418 prefix_len + 1, *buffer_size);
e41ef6fd
ZJS		 419 /* One lz4 < 1.8.3, we might get "failure" (r < 0), or "success" where
420 * just a part of the buffer is decompressed. But if we get a smaller
421 * amount of bytes than requested, we don't know whether there isn't enough
422 * data to fill the requested size or whether we just got a partial answer.
423 */
424 if (r < 0 || (size_t) r < prefix_len + 1) {
425 size_t size;
426
427 if (LZ4_versionNumber() >= 10803)
428 /* We trust that the newer lz4 decompresses the number of bytes we
429 * requested if available in the compressed string. */
430 return 0;
431
432 if (r > 0)
433 /* Compare what we have first, in case of mismatch we can
434 * shortcut the full comparison. */
435 if (memcmp(*buffer, prefix, r) != 0)
436 return 0;
437
438 /* Before version 1.8.3, lz4 always tries to decode full a "sequence",
439 * so in pathological cases might need to decompress the full field. */
1f4b467d
ZJS		 440 r = decompress_blob_lz4(src, src_size, buffer, buffer_size, &size, 0);
441 if (r < 0)
442 return r;
d89c8fdf 443
e41ef6fd
ZJS		 444 if (size < prefix_len + 1)
445 return 0;
446 }
d89c8fdf 447
e41ef6fd
ZJS		 448 return memcmp(*buffer, prefix, prefix_len) == 0 &&
449 ((const uint8_t*) *buffer)[prefix_len] == extra;
d89c8fdf
ZJS		 450#else
451 return -EPROTONOSUPPORT;
452#endif
e4e61fdb 453}
355b59e2 454
8653185a
LP		 455int decompress_startswith_zstd(
456 const void *src, uint64_t src_size,
457 void **buffer, size_t *buffer_size,
458 const void *prefix, size_t prefix_len,
459 uint8_t extra) {
460#if HAVE_ZSTD
8653185a
LP		 461 assert(src);
462 assert(src_size > 0);
463 assert(buffer);
464 assert(buffer_size);
465 assert(prefix);
466 assert(*buffer_size == 0 || *buffer);
467
e4a321fc
ZJS		 468 uint64_t size = ZSTD_getFrameContentSize(src, src_size);
469 if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
470 return -EBADMSG;
471
472 if (size < prefix_len + 1)
473 return 0; /* Decompressed text too short to match the prefix and extra */
474
475 _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx();
8653185a
LP		 476 if (!dctx)
477 return -ENOMEM;
478
479 if (!(greedy_realloc(buffer, buffer_size, MAX(ZSTD_DStreamOutSize(), prefix_len + 1), 1)))
480 return -ENOMEM;
481
482 ZSTD_inBuffer input = {
483 .src = src,
484 .size = src_size,
485 };
486 ZSTD_outBuffer output = {
487 .dst = *buffer,
488 .size = *buffer_size,
489 };
e4a321fc 490 size_t k;
8653185a 491
e4a321fc
ZJS		 492 k = ZSTD_decompressStream(dctx, &output, &input);
493 if (ZSTD_isError(k)) {
494 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k));
495 return zstd_ret_to_errno(k);
8653185a 496 }
e4a321fc
ZJS		 497 assert(output.pos >= prefix_len + 1);
498
499 return memcmp(*buffer, prefix, prefix_len) == 0 &&
500 ((const uint8_t*) *buffer)[prefix_len] == extra;
8653185a
LP		 501#else
502 return -EPROTONOSUPPORT;
503#endif
504}
505
506int decompress_startswith(
507 int compression,
508 const void *src, uint64_t src_size,
509 void **buffer, size_t *buffer_size,
510 const void *prefix, size_t prefix_len,
511 uint8_t extra) {
512
d89c8fdf 513 if (compression == OBJECT_COMPRESSED_XZ)
8653185a
LP		 514 return decompress_startswith_xz(
515 src, src_size,
516 buffer, buffer_size,
517 prefix, prefix_len,
518 extra);
519
d89c8fdf 520 else if (compression == OBJECT_COMPRESSED_LZ4)
8653185a
LP		 521 return decompress_startswith_lz4(
522 src, src_size,
523 buffer, buffer_size,
524 prefix, prefix_len,
525 extra);
526 else if (compression == OBJECT_COMPRESSED_ZSTD)
527 return decompress_startswith_zstd(
528 src, src_size,
529 buffer, buffer_size,
530 prefix, prefix_len,
531 extra);
d89c8fdf
ZJS		 532 else
533 return -EBADMSG;
534}
535
59f448cf 536int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
349cc4a5 537#if HAVE_XZ
355b59e2
ZJS		 538 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
539 lzma_ret ret;
355b59e2
ZJS		 540 uint8_t buf[BUFSIZ], out[BUFSIZ];
541 lzma_action action = LZMA_RUN;
542
543 assert(fdf >= 0);
544 assert(fdt >= 0);
545
d89c8fdf 546 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
d7a0f1f4
FS		 547 if (ret != LZMA_OK)
548 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
549 "Failed to initialize XZ encoder: code %u",
550 ret);
355b59e2
ZJS		 551
552 for (;;) {
553 if (s.avail_in == 0 && action == LZMA_RUN) {
554 size_t m = sizeof(buf);
555 ssize_t n;
556
f5fbe71d 557 if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes)
59f448cf 558 m = (size_t) max_bytes;
355b59e2
ZJS		 559
560 n = read(fdf, buf, m);
561 if (n < 0)
562 return -errno;
563 if (n == 0)
564 action = LZMA_FINISH;
565 else {
566 s.next_in = buf;
567 s.avail_in = n;
568
f5fbe71d 569 if (max_bytes != UINT64_MAX) {
59f448cf 570 assert(max_bytes >= (uint64_t) n);
355b59e2
ZJS		 571 max_bytes -= n;
572 }
573 }
574 }
575
576 if (s.avail_out == 0) {
577 s.next_out = out;
578 s.avail_out = sizeof(out);
579 }
580
581 ret = lzma_code(&s, action);
d7a0f1f4
FS		 582 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
583 return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
584 "Compression failed: code %u",
585 ret);
355b59e2
ZJS		 586
587 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
588 ssize_t n, k;
589
590 n = sizeof(out) - s.avail_out;
591
355b59e2
ZJS		 592 k = loop_write(fdt, out, n, false);
593 if (k < 0)
594 return k;
355b59e2
ZJS		 595
596 if (ret == LZMA_STREAM_END) {
fa1c4b51 597 log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
355b59e2
ZJS		 598 s.total_in, s.total_out,
599 (double) s.total_out / s.total_in * 100);
600
601 return 0;
602 }
603 }
604 }
3b1a55e1
ZJS		 605#else
606 return -EPROTONOSUPPORT;
607#endif
355b59e2
ZJS		 608}
609
4b5bc539 610#define LZ4_BUFSIZE (512*1024u)
d89c8fdf 611
59f448cf 612int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
d89c8fdf 613
349cc4a5 614#if HAVE_LZ4
4b5bc539
ZJS		 615 LZ4F_errorCode_t c;
616 _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
617 _cleanup_free_ char *buf = NULL;
618 char *src = NULL;
d487b815 619 size_t size, n, total_in = 0, total_out, offset = 0, frame_size;
4b5bc539
ZJS		 620 struct stat st;
621 int r;
622 static const LZ4F_compressOptions_t options = {
623 .stableSrc = 1,
624 };
625 static const LZ4F_preferences_t preferences = {
626 .frameInfo.blockSizeID = 5,
627 };
d89c8fdf 628
4b5bc539
ZJS		 629 c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
630 if (LZ4F_isError(c))
631 return -ENOMEM;
d89c8fdf 632
4b5bc539 633 if (fstat(fdf, &st) < 0)
5146f9f0 634 return log_debug_errno(errno, "fstat() failed: %m");
d89c8fdf 635
4b5bc539
ZJS		 636 frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
637 size = frame_size + 64*1024; /* add some space for header and trailer */
638 buf = malloc(size);
639 if (!buf)
640 return -ENOMEM;
d89c8fdf 641
d487b815 642 n = offset = total_out = LZ4F_compressBegin(ctx, buf, size, &preferences);
4b5bc539
ZJS		 643 if (LZ4F_isError(n))
644 return -EINVAL;
d89c8fdf 645
4b5bc539 646 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0);
e0685172 647 if (src == MAP_FAILED)
4b5bc539 648 return -errno;
d89c8fdf 649
4b5bc539 650 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size, n);
d89c8fdf 651
4b5bc539
ZJS		 652 while (total_in < (size_t) st.st_size) {
653 ssize_t k;
d89c8fdf 654
4b5bc539
ZJS		 655 k = MIN(LZ4_BUFSIZE, st.st_size - total_in);
656 n = LZ4F_compressUpdate(ctx, buf + offset, size - offset,
657 src + total_in, k, &options);
658 if (LZ4F_isError(n)) {
659 r = -ENOTRECOVERABLE;
660 goto cleanup;
d89c8fdf
ZJS		 661 }
662
4b5bc539
ZJS		 663 total_in += k;
664 offset += n;
665 total_out += n;
d89c8fdf 666
f5fbe71d 667 if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes) {
363729c4
ZJS		 668 r = log_debug_errno(SYNTHETIC_ERRNO(EFBIG),
669 "Compressed stream longer than %" PRIu64 " bytes", max_bytes);
670 goto cleanup;
671 }
d89c8fdf 672
4b5bc539 673 if (size - offset < frame_size + 4) {
4b5bc539
ZJS		 674 k = loop_write(fdt, buf, offset, false);
675 if (k < 0) {
676 r = k;
677 goto cleanup;
678 }
679 offset = 0;
680 }
681 }
d89c8fdf 682
4b5bc539
ZJS		 683 n = LZ4F_compressEnd(ctx, buf + offset, size - offset, &options);
684 if (LZ4F_isError(n)) {
685 r = -ENOTRECOVERABLE;
686 goto cleanup;
d89c8fdf
ZJS		 687 }
688
4b5bc539
ZJS		 689 offset += n;
690 total_out += n;
4b5bc539
ZJS		 691 r = loop_write(fdt, buf, offset, false);
692 if (r < 0)
693 goto cleanup;
d89c8fdf
ZJS		 694
695 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
696 total_in, total_out,
697 (double) total_out / total_in * 100);
4b5bc539
ZJS		 698 cleanup:
699 munmap(src, st.st_size);
700 return r;
d89c8fdf
ZJS		 701#else
702 return -EPROTONOSUPPORT;
703#endif
704}
705
59f448cf 706int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
d89c8fdf 707
349cc4a5 708#if HAVE_XZ
355b59e2
ZJS		 709 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
710 lzma_ret ret;
711
712 uint8_t buf[BUFSIZ], out[BUFSIZ];
713 lzma_action action = LZMA_RUN;
714
715 assert(fdf >= 0);
716 assert(fdt >= 0);
717
718 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
d7a0f1f4
FS		 719 if (ret != LZMA_OK)
720 return log_debug_errno(SYNTHETIC_ERRNO(ENOMEM),
721 "Failed to initialize XZ decoder: code %u",
722 ret);
355b59e2
ZJS		 723
724 for (;;) {
725 if (s.avail_in == 0 && action == LZMA_RUN) {
726 ssize_t n;
727
728 n = read(fdf, buf, sizeof(buf));
729 if (n < 0)
730 return -errno;
731 if (n == 0)
732 action = LZMA_FINISH;
733 else {
734 s.next_in = buf;
735 s.avail_in = n;
736 }
737 }
738
739 if (s.avail_out == 0) {
740 s.next_out = out;
741 s.avail_out = sizeof(out);
742 }
743
744 ret = lzma_code(&s, action);
d7a0f1f4
FS		 745 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
746 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
747 "Decompression failed: code %u",
748 ret);
355b59e2
ZJS		 749
750 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
751 ssize_t n, k;
752
753 n = sizeof(out) - s.avail_out;
754
f5fbe71d 755 if (max_bytes != UINT64_MAX) {
59f448cf 756 if (max_bytes < (uint64_t) n)
d89c8fdf 757 return -EFBIG;
355b59e2
ZJS		 758
759 max_bytes -= n;
760 }
761
355b59e2
ZJS		 762 k = loop_write(fdt, out, n, false);
763 if (k < 0)
764 return k;
355b59e2
ZJS		 765
766 if (ret == LZMA_STREAM_END) {
fa1c4b51 767 log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
355b59e2
ZJS		 768 s.total_in, s.total_out,
769 (double) s.total_out / s.total_in * 100);
770
771 return 0;
772 }
773 }
774 }
d89c8fdf 775#else
d7a0f1f4
FS		 776 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
777 "Cannot decompress file. Compiled without XZ support.");
d89c8fdf
ZJS		 778#endif
779}
780
8e64dd1e 781int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
349cc4a5 782#if HAVE_LZ4
4b5bc539
ZJS		 783 size_t c;
784 _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
785 _cleanup_free_ char *buf = NULL;
786 char *src;
787 struct stat st;
788 int r = 0;
789 size_t total_in = 0, total_out = 0;
790
791 c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
792 if (LZ4F_isError(c))
793 return -ENOMEM;
794
795 if (fstat(in, &st) < 0)
5146f9f0 796 return log_debug_errno(errno, "fstat() failed: %m");
4b5bc539
ZJS		 797
798 buf = malloc(LZ4_BUFSIZE);
799 if (!buf)
800 return -ENOMEM;
801
802 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in, 0);
e0685172 803 if (src == MAP_FAILED)
4b5bc539
ZJS		 804 return -errno;
805
806 while (total_in < (size_t) st.st_size) {
807 size_t produced = LZ4_BUFSIZE;
808 size_t used = st.st_size - total_in;
809
810 c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
811 if (LZ4F_isError(c)) {
812 r = -EBADMSG;
813 goto cleanup;
814 }
815
816 total_in += used;
817 total_out += produced;
818
f5fbe71d 819 if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes) {
82e24b00 820 log_debug("Decompressed stream longer than %"PRIu64" bytes", max_bytes);
5146f9f0 821 r = -EFBIG;
4b5bc539
ZJS		 822 goto cleanup;
823 }
824
825 r = loop_write(out, buf, produced, false);
826 if (r < 0)
827 goto cleanup;
828 }
829
830 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
831 total_in, total_out,
25048348 832 total_in > 0 ? (double) total_out / total_in * 100 : 0.0);
4b5bc539
ZJS		 833 cleanup:
834 munmap(src, st.st_size);
835 return r;
d89c8fdf 836#else
d7a0f1f4
FS		 837 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
838 "Cannot decompress file. Compiled without LZ4 support.");
d89c8fdf
ZJS		 839#endif
840}
841
ef5924aa
NL		 842int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
843#if HAVE_ZSTD
844 _cleanup_(ZSTD_freeCCtxp) ZSTD_CCtx *cctx = NULL;
845 _cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
846 size_t in_allocsize, out_allocsize;
847 size_t z;
848 uint64_t left = max_bytes, in_bytes = 0;
ef5924aa
NL		 849
850 assert(fdf >= 0);
851 assert(fdt >= 0);
852
853 /* Create the context and buffers */
854 in_allocsize = ZSTD_CStreamInSize();
855 out_allocsize = ZSTD_CStreamOutSize();
856 in_buff = malloc(in_allocsize);
857 out_buff = malloc(out_allocsize);
858 cctx = ZSTD_createCCtx();
859 if (!cctx || !out_buff || !in_buff)
860 return -ENOMEM;
861
ef5924aa
NL		 862 z = ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1);
863 if (ZSTD_isError(z))
864 log_debug("Failed to enable ZSTD checksum, ignoring: %s", ZSTD_getErrorName(z));
865
866 /* This loop read from the input file, compresses that entire chunk,
867 * and writes all output produced to the output file.
868 */
869 for (;;) {
870 bool is_last_chunk;
871 ZSTD_inBuffer input = {
872 .src = in_buff,
873 .size = 0,
874 .pos = 0
875 };
876 ssize_t red;
877
878 red = loop_read(fdf, in_buff, in_allocsize, true);
879 if (red < 0)
880 return red;
881 is_last_chunk = red == 0;
882
883 in_bytes += (size_t) red;
884 input.size = (size_t) red;
885
886 for (bool finished = false; !finished;) {
887 ZSTD_outBuffer output = {
888 .dst = out_buff,
889 .size = out_allocsize,
890 .pos = 0
891 };
892 size_t remaining;
893 ssize_t wrote;
894
895 /* Compress into the output buffer and write all of the
896 * output to the file so we can reuse the buffer next
897 * iteration.
898 */
899 remaining = ZSTD_compressStream2(
900 cctx, &output, &input,
901 is_last_chunk ? ZSTD_e_end : ZSTD_e_continue);
902
903 if (ZSTD_isError(remaining)) {
904 log_debug("ZSTD encoder failed: %s", ZSTD_getErrorName(remaining));
905 return zstd_ret_to_errno(remaining);
906 }
907
908 if (left < output.pos)
909 return -EFBIG;
910
911 wrote = loop_write(fdt, output.dst, output.pos, 1);
912 if (wrote < 0)
913 return wrote;
914
915 left -= output.pos;
916
917 /* If we're on the last chunk we're finished when zstd
918 * returns 0, which means its consumed all the input AND
919 * finished the frame. Otherwise, we're finished when
920 * we've consumed all the input.
921 */
922 finished = is_last_chunk ? (remaining == 0) : (input.pos == input.size);
923 }
924
925 /* zstd only returns 0 when the input is completely consumed */
926 assert(input.pos == input.size);
927 if (is_last_chunk)
928 break;
929 }
930
89d36ce8
YW		 931 if (in_bytes > 0)
932 log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
933 in_bytes, max_bytes - left, (double) (max_bytes - left) / in_bytes * 100);
934 else
935 log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes)",
936 in_bytes, max_bytes - left);
ef5924aa
NL		 937
938 return 0;
939#else
940 return -EPROTONOSUPPORT;
941#endif
942}
943
944int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
945#if HAVE_ZSTD
946 _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
947 _cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
948 size_t in_allocsize, out_allocsize;
949 size_t last_result = 0;
950 uint64_t left = max_bytes, in_bytes = 0;
951
952 assert(fdf >= 0);
953 assert(fdt >= 0);
954
955 /* Create the context and buffers */
956 in_allocsize = ZSTD_DStreamInSize();
957 out_allocsize = ZSTD_DStreamOutSize();
958 in_buff = malloc(in_allocsize);
959 out_buff = malloc(out_allocsize);
960 dctx = ZSTD_createDCtx();
961 if (!dctx || !out_buff || !in_buff)
962 return -ENOMEM;
963
964 /* This loop assumes that the input file is one or more concatenated
965 * zstd streams. This example won't work if there is trailing non-zstd
966 * data at the end, but streaming decompression in general handles this
967 * case. ZSTD_decompressStream() returns 0 exactly when the frame is
968 * completed, and doesn't consume input after the frame.
969 */
970 for (;;) {
971 bool has_error = false;
972 ZSTD_inBuffer input = {
973 .src = in_buff,
974 .size = 0,
975 .pos = 0
976 };
977 ssize_t red;
978
979 red = loop_read(fdf, in_buff, in_allocsize, true);
980 if (red < 0)
981 return red;
982 if (red == 0)
983 break;
984
985 in_bytes += (size_t) red;
986 input.size = (size_t) red;
987 input.pos = 0;
988
989 /* Given a valid frame, zstd won't consume the last byte of the
990 * frame until it has flushed all of the decompressed data of
991 * the frame. So input.pos < input.size means frame is not done
992 * or there is still output available.
993 */
994 while (input.pos < input.size) {
995 ZSTD_outBuffer output = {
996 .dst = out_buff,
997 .size = out_allocsize,
998 .pos = 0
999 };
1000 ssize_t wrote;
1001 /* The return code is zero if the frame is complete, but
1002 * there may be multiple frames concatenated together.
1003 * Zstd will automatically reset the context when a
1004 * frame is complete. Still, calling ZSTD_DCtx_reset()
1005 * can be useful to reset the context to a clean state,
1006 * for instance if the last decompression call returned
1007 * an error.
1008 */
1009 last_result = ZSTD_decompressStream(dctx, &output, &input);
1010 if (ZSTD_isError(last_result)) {
1011 has_error = true;
1012 break;
1013 }
1014
1015 if (left < output.pos)
1016 return -EFBIG;
1017
1018 wrote = loop_write(fdt, output.dst, output.pos, 1);
1019 if (wrote < 0)
1020 return wrote;
1021
1022 left -= output.pos;
1023 }
1024 if (has_error)
1025 break;
1026 }
1027
1028 if (in_bytes == 0)
1029 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD decoder failed: no data read");
1030
1031 if (last_result != 0) {
1032 /* The last return value from ZSTD_decompressStream did not end
1033 * on a frame, but we reached the end of the file! We assume
1034 * this is an error, and the input was truncated.
1035 */
1036 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(last_result));
1037 return zstd_ret_to_errno(last_result);
1038 }
1039
1040 log_debug(
1041 "ZSTD decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
1042 in_bytes,
1043 max_bytes - left,
1044 (double) (max_bytes - left) / in_bytes * 100);
1045 return 0;
1046#else
d7a0f1f4
FS		 1047 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
1048 "Cannot decompress file. Compiled without ZSTD support.");
ef5924aa
NL		 1049#endif
1050}
1051
59f448cf 1052int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
d89c8fdf
ZJS		 1053
1054 if (endswith(filename, ".lz4"))
1055 return decompress_stream_lz4(fdf, fdt, max_bytes);
1056 else if (endswith(filename, ".xz"))
1057 return decompress_stream_xz(fdf, fdt, max_bytes);
ef5924aa
NL		 1058 else if (endswith(filename, ".zst"))
1059 return decompress_stream_zstd(fdf, fdt, max_bytes);
d89c8fdf
ZJS		 1060 else
1061 return -EPROTONOSUPPORT;
355b59e2 1062}
Unnamed repository; edit this file 'description' to name the repository.