]> git.ipfire.org Git - thirdparty/systemd.git/blob - src/journal/compress.c
Use `PRIu64` to print `uint64_t` in log msgs
[thirdparty/systemd.git] / src / journal / compress.c
1 /***
2 This file is part of systemd.
3
4 Copyright 2011 Lennart Poettering
5
6 systemd is free software; you can redistribute it and/or modify it
7 under the terms of the GNU Lesser General Public License as published by
8 the Free Software Foundation; either version 2.1 of the License, or
9 (at your option) any later version.
10
11 systemd is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public License
17 along with systemd; If not, see <http://www.gnu.org/licenses/>.
18 ***/
19
20 #include <inttypes.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <sys/mman.h>
24 #include <unistd.h>
25
26 #ifdef HAVE_XZ
27 #include <lzma.h>
28 #endif
29
30 #ifdef HAVE_LZ4
31 #include <lz4.h>
32 #include <lz4frame.h>
33 #endif
34
35 #include "alloc-util.h"
36 #include "compress.h"
37 #include "fd-util.h"
38 #include "io-util.h"
39 #include "journal-def.h"
40 #include "macro.h"
41 #include "sparse-endian.h"
42 #include "string-table.h"
43 #include "string-util.h"
44 #include "util.h"
45
46 #ifdef HAVE_LZ4
47 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t, LZ4F_freeCompressionContext);
48 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext);
49 #endif
50
51 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
52
53 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
54 [OBJECT_COMPRESSED_XZ] = "XZ",
55 [OBJECT_COMPRESSED_LZ4] = "LZ4",
56 };
57
58 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
59
60 int compress_blob_xz(const void *src, uint64_t src_size,
61 void *dst, size_t dst_alloc_size, size_t *dst_size) {
62 #ifdef HAVE_XZ
63 static const lzma_options_lzma opt = {
64 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
65 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
66 };
67 static const lzma_filter filters[] = {
68 { LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt },
69 { LZMA_VLI_UNKNOWN, NULL }
70 };
71 lzma_ret ret;
72 size_t out_pos = 0;
73
74 assert(src);
75 assert(src_size > 0);
76 assert(dst);
77 assert(dst_alloc_size > 0);
78 assert(dst_size);
79
80 /* Returns < 0 if we couldn't compress the data or the
81 * compressed result is longer than the original */
82
83 if (src_size < 80)
84 return -ENOBUFS;
85
86 ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
87 src, src_size, dst, &out_pos, dst_alloc_size);
88 if (ret != LZMA_OK)
89 return -ENOBUFS;
90
91 *dst_size = out_pos;
92 return 0;
93 #else
94 return -EPROTONOSUPPORT;
95 #endif
96 }
97
98 int compress_blob_lz4(const void *src, uint64_t src_size,
99 void *dst, size_t dst_alloc_size, size_t *dst_size) {
100 #ifdef HAVE_LZ4
101 int r;
102
103 assert(src);
104 assert(src_size > 0);
105 assert(dst);
106 assert(dst_alloc_size > 0);
107 assert(dst_size);
108
109 /* Returns < 0 if we couldn't compress the data or the
110 * compressed result is longer than the original */
111
112 if (src_size < 9)
113 return -ENOBUFS;
114
115 r = LZ4_compress_limitedOutput(src, dst + 8, src_size, (int) dst_alloc_size - 8);
116 if (r <= 0)
117 return -ENOBUFS;
118
119 *(le64_t*) dst = htole64(src_size);
120 *dst_size = r + 8;
121
122 return 0;
123 #else
124 return -EPROTONOSUPPORT;
125 #endif
126 }
127
128
129 int decompress_blob_xz(const void *src, uint64_t src_size,
130 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
131
132 #ifdef HAVE_XZ
133 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
134 lzma_ret ret;
135 size_t space;
136
137 assert(src);
138 assert(src_size > 0);
139 assert(dst);
140 assert(dst_alloc_size);
141 assert(dst_size);
142 assert(*dst_alloc_size == 0 || *dst);
143
144 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
145 if (ret != LZMA_OK)
146 return -ENOMEM;
147
148 space = MIN(src_size * 2, dst_max ?: (size_t) -1);
149 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
150 return -ENOMEM;
151
152 s.next_in = src;
153 s.avail_in = src_size;
154
155 s.next_out = *dst;
156 s.avail_out = space;
157
158 for (;;) {
159 size_t used;
160
161 ret = lzma_code(&s, LZMA_FINISH);
162
163 if (ret == LZMA_STREAM_END)
164 break;
165 else if (ret != LZMA_OK)
166 return -ENOMEM;
167
168 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
169 break;
170 else if (dst_max > 0 && space == dst_max)
171 return -ENOBUFS;
172
173 used = space - s.avail_out;
174 space = MIN(2 * space, dst_max ?: (size_t) -1);
175 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
176 return -ENOMEM;
177
178 s.avail_out = space - used;
179 s.next_out = *dst + used;
180 }
181
182 *dst_size = space - s.avail_out;
183 return 0;
184 #else
185 return -EPROTONOSUPPORT;
186 #endif
187 }
188
189 int decompress_blob_lz4(const void *src, uint64_t src_size,
190 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
191
192 #ifdef HAVE_LZ4
193 char* out;
194 int r, size; /* LZ4 uses int for size */
195
196 assert(src);
197 assert(src_size > 0);
198 assert(dst);
199 assert(dst_alloc_size);
200 assert(dst_size);
201 assert(*dst_alloc_size == 0 || *dst);
202
203 if (src_size <= 8)
204 return -EBADMSG;
205
206 size = le64toh( *(le64_t*)src );
207 if (size < 0 || (unsigned) size != le64toh(*(le64_t*)src))
208 return -EFBIG;
209 if ((size_t) size > *dst_alloc_size) {
210 out = realloc(*dst, size);
211 if (!out)
212 return -ENOMEM;
213 *dst = out;
214 *dst_alloc_size = size;
215 } else
216 out = *dst;
217
218 r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
219 if (r < 0 || r != size)
220 return -EBADMSG;
221
222 *dst_size = size;
223 return 0;
224 #else
225 return -EPROTONOSUPPORT;
226 #endif
227 }
228
229 int decompress_blob(int compression,
230 const void *src, uint64_t src_size,
231 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
232 if (compression == OBJECT_COMPRESSED_XZ)
233 return decompress_blob_xz(src, src_size,
234 dst, dst_alloc_size, dst_size, dst_max);
235 else if (compression == OBJECT_COMPRESSED_LZ4)
236 return decompress_blob_lz4(src, src_size,
237 dst, dst_alloc_size, dst_size, dst_max);
238 else
239 return -EBADMSG;
240 }
241
242
243 int decompress_startswith_xz(const void *src, uint64_t src_size,
244 void **buffer, size_t *buffer_size,
245 const void *prefix, size_t prefix_len,
246 uint8_t extra) {
247
248 #ifdef HAVE_XZ
249 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
250 lzma_ret ret;
251
252 /* Checks whether the decompressed blob starts with the
253 * mentioned prefix. The byte extra needs to follow the
254 * prefix */
255
256 assert(src);
257 assert(src_size > 0);
258 assert(buffer);
259 assert(buffer_size);
260 assert(prefix);
261 assert(*buffer_size == 0 || *buffer);
262
263 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
264 if (ret != LZMA_OK)
265 return -EBADMSG;
266
267 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
268 return -ENOMEM;
269
270 s.next_in = src;
271 s.avail_in = src_size;
272
273 s.next_out = *buffer;
274 s.avail_out = *buffer_size;
275
276 for (;;) {
277 ret = lzma_code(&s, LZMA_FINISH);
278
279 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
280 return -EBADMSG;
281
282 if (*buffer_size - s.avail_out >= prefix_len + 1)
283 return memcmp(*buffer, prefix, prefix_len) == 0 &&
284 ((const uint8_t*) *buffer)[prefix_len] == extra;
285
286 if (ret == LZMA_STREAM_END)
287 return 0;
288
289 s.avail_out += *buffer_size;
290
291 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
292 return -ENOMEM;
293
294 s.next_out = *buffer + *buffer_size - s.avail_out;
295 }
296
297 #else
298 return -EPROTONOSUPPORT;
299 #endif
300 }
301
302 int decompress_startswith_lz4(const void *src, uint64_t src_size,
303 void **buffer, size_t *buffer_size,
304 const void *prefix, size_t prefix_len,
305 uint8_t extra) {
306 #ifdef HAVE_LZ4
307 /* Checks whether the decompressed blob starts with the
308 * mentioned prefix. The byte extra needs to follow the
309 * prefix */
310
311 int r;
312 size_t size;
313
314 assert(src);
315 assert(src_size > 0);
316 assert(buffer);
317 assert(buffer_size);
318 assert(prefix);
319 assert(*buffer_size == 0 || *buffer);
320
321 if (src_size <= 8)
322 return -EBADMSG;
323
324 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
325 return -ENOMEM;
326
327 r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
328 prefix_len + 1, *buffer_size);
329 if (r >= 0)
330 size = (unsigned) r;
331 else {
332 /* lz4 always tries to decode full "sequence", so in
333 * pathological cases might need to decompress the
334 * full field. */
335 r = decompress_blob_lz4(src, src_size, buffer, buffer_size, &size, 0);
336 if (r < 0)
337 return r;
338 }
339
340 if (size >= prefix_len + 1)
341 return memcmp(*buffer, prefix, prefix_len) == 0 &&
342 ((const uint8_t*) *buffer)[prefix_len] == extra;
343 else
344 return 0;
345
346 #else
347 return -EPROTONOSUPPORT;
348 #endif
349 }
350
351 int decompress_startswith(int compression,
352 const void *src, uint64_t src_size,
353 void **buffer, size_t *buffer_size,
354 const void *prefix, size_t prefix_len,
355 uint8_t extra) {
356 if (compression == OBJECT_COMPRESSED_XZ)
357 return decompress_startswith_xz(src, src_size,
358 buffer, buffer_size,
359 prefix, prefix_len,
360 extra);
361 else if (compression == OBJECT_COMPRESSED_LZ4)
362 return decompress_startswith_lz4(src, src_size,
363 buffer, buffer_size,
364 prefix, prefix_len,
365 extra);
366 else
367 return -EBADMSG;
368 }
369
370 int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
371 #ifdef HAVE_XZ
372 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
373 lzma_ret ret;
374 uint8_t buf[BUFSIZ], out[BUFSIZ];
375 lzma_action action = LZMA_RUN;
376
377 assert(fdf >= 0);
378 assert(fdt >= 0);
379
380 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
381 if (ret != LZMA_OK) {
382 log_error("Failed to initialize XZ encoder: code %u", ret);
383 return -EINVAL;
384 }
385
386 for (;;) {
387 if (s.avail_in == 0 && action == LZMA_RUN) {
388 size_t m = sizeof(buf);
389 ssize_t n;
390
391 if (max_bytes != (uint64_t) -1 && (uint64_t) m > max_bytes)
392 m = (size_t) max_bytes;
393
394 n = read(fdf, buf, m);
395 if (n < 0)
396 return -errno;
397 if (n == 0)
398 action = LZMA_FINISH;
399 else {
400 s.next_in = buf;
401 s.avail_in = n;
402
403 if (max_bytes != (uint64_t) -1) {
404 assert(max_bytes >= (uint64_t) n);
405 max_bytes -= n;
406 }
407 }
408 }
409
410 if (s.avail_out == 0) {
411 s.next_out = out;
412 s.avail_out = sizeof(out);
413 }
414
415 ret = lzma_code(&s, action);
416 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
417 log_error("Compression failed: code %u", ret);
418 return -EBADMSG;
419 }
420
421 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
422 ssize_t n, k;
423
424 n = sizeof(out) - s.avail_out;
425
426 k = loop_write(fdt, out, n, false);
427 if (k < 0)
428 return k;
429
430 if (ret == LZMA_STREAM_END) {
431 log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
432 s.total_in, s.total_out,
433 (double) s.total_out / s.total_in * 100);
434
435 return 0;
436 }
437 }
438 }
439 #else
440 return -EPROTONOSUPPORT;
441 #endif
442 }
443
444 #define LZ4_BUFSIZE (512*1024u)
445
446 int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
447
448 #ifdef HAVE_LZ4
449 LZ4F_errorCode_t c;
450 _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
451 _cleanup_free_ char *buf = NULL;
452 char *src = NULL;
453 size_t size, n, total_in = 0, total_out, offset = 0, frame_size;
454 struct stat st;
455 int r;
456 static const LZ4F_compressOptions_t options = {
457 .stableSrc = 1,
458 };
459 static const LZ4F_preferences_t preferences = {
460 .frameInfo.blockSizeID = 5,
461 };
462
463 c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
464 if (LZ4F_isError(c))
465 return -ENOMEM;
466
467 if (fstat(fdf, &st) < 0)
468 return log_debug_errno(errno, "fstat() failed: %m");
469
470 frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
471 size = frame_size + 64*1024; /* add some space for header and trailer */
472 buf = malloc(size);
473 if (!buf)
474 return -ENOMEM;
475
476 n = offset = total_out = LZ4F_compressBegin(ctx, buf, size, &preferences);
477 if (LZ4F_isError(n))
478 return -EINVAL;
479
480 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0);
481 if (src == MAP_FAILED)
482 return -errno;
483
484 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size, n);
485
486 while (total_in < (size_t) st.st_size) {
487 ssize_t k;
488
489 k = MIN(LZ4_BUFSIZE, st.st_size - total_in);
490 n = LZ4F_compressUpdate(ctx, buf + offset, size - offset,
491 src + total_in, k, &options);
492 if (LZ4F_isError(n)) {
493 r = -ENOTRECOVERABLE;
494 goto cleanup;
495 }
496
497 total_in += k;
498 offset += n;
499 total_out += n;
500
501 if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) {
502 log_debug("Compressed stream longer than %"PRIu64" bytes", max_bytes);
503 return -EFBIG;
504 }
505
506 if (size - offset < frame_size + 4) {
507 k = loop_write(fdt, buf, offset, false);
508 if (k < 0) {
509 r = k;
510 goto cleanup;
511 }
512 offset = 0;
513 }
514 }
515
516 n = LZ4F_compressEnd(ctx, buf + offset, size - offset, &options);
517 if (LZ4F_isError(n)) {
518 r = -ENOTRECOVERABLE;
519 goto cleanup;
520 }
521
522 offset += n;
523 total_out += n;
524 r = loop_write(fdt, buf, offset, false);
525 if (r < 0)
526 goto cleanup;
527
528 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
529 total_in, total_out,
530 (double) total_out / total_in * 100);
531 cleanup:
532 munmap(src, st.st_size);
533 return r;
534 #else
535 return -EPROTONOSUPPORT;
536 #endif
537 }
538
539 int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
540
541 #ifdef HAVE_XZ
542 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
543 lzma_ret ret;
544
545 uint8_t buf[BUFSIZ], out[BUFSIZ];
546 lzma_action action = LZMA_RUN;
547
548 assert(fdf >= 0);
549 assert(fdt >= 0);
550
551 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
552 if (ret != LZMA_OK) {
553 log_debug("Failed to initialize XZ decoder: code %u", ret);
554 return -ENOMEM;
555 }
556
557 for (;;) {
558 if (s.avail_in == 0 && action == LZMA_RUN) {
559 ssize_t n;
560
561 n = read(fdf, buf, sizeof(buf));
562 if (n < 0)
563 return -errno;
564 if (n == 0)
565 action = LZMA_FINISH;
566 else {
567 s.next_in = buf;
568 s.avail_in = n;
569 }
570 }
571
572 if (s.avail_out == 0) {
573 s.next_out = out;
574 s.avail_out = sizeof(out);
575 }
576
577 ret = lzma_code(&s, action);
578 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
579 log_debug("Decompression failed: code %u", ret);
580 return -EBADMSG;
581 }
582
583 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
584 ssize_t n, k;
585
586 n = sizeof(out) - s.avail_out;
587
588 if (max_bytes != (uint64_t) -1) {
589 if (max_bytes < (uint64_t) n)
590 return -EFBIG;
591
592 max_bytes -= n;
593 }
594
595 k = loop_write(fdt, out, n, false);
596 if (k < 0)
597 return k;
598
599 if (ret == LZMA_STREAM_END) {
600 log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
601 s.total_in, s.total_out,
602 (double) s.total_out / s.total_in * 100);
603
604 return 0;
605 }
606 }
607 }
608 #else
609 log_debug("Cannot decompress file. Compiled without XZ support.");
610 return -EPROTONOSUPPORT;
611 #endif
612 }
613
614 int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
615 #ifdef HAVE_LZ4
616 size_t c;
617 _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
618 _cleanup_free_ char *buf = NULL;
619 char *src;
620 struct stat st;
621 int r = 0;
622 size_t total_in = 0, total_out = 0;
623
624 c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
625 if (LZ4F_isError(c))
626 return -ENOMEM;
627
628 if (fstat(in, &st) < 0)
629 return log_debug_errno(errno, "fstat() failed: %m");
630
631 buf = malloc(LZ4_BUFSIZE);
632 if (!buf)
633 return -ENOMEM;
634
635 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in, 0);
636 if (src == MAP_FAILED)
637 return -errno;
638
639 while (total_in < (size_t) st.st_size) {
640 size_t produced = LZ4_BUFSIZE;
641 size_t used = st.st_size - total_in;
642
643 c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
644 if (LZ4F_isError(c)) {
645 r = -EBADMSG;
646 goto cleanup;
647 }
648
649 total_in += used;
650 total_out += produced;
651
652 if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) {
653 log_debug("Decompressed stream longer than %"PRIu64" bytes", max_bytes);
654 r = -EFBIG;
655 goto cleanup;
656 }
657
658 r = loop_write(out, buf, produced, false);
659 if (r < 0)
660 goto cleanup;
661 }
662
663 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
664 total_in, total_out,
665 (double) total_out / total_in * 100);
666 cleanup:
667 munmap(src, st.st_size);
668 return r;
669 #else
670 log_debug("Cannot decompress file. Compiled without LZ4 support.");
671 return -EPROTONOSUPPORT;
672 #endif
673 }
674
675 int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
676
677 if (endswith(filename, ".lz4"))
678 return decompress_stream_lz4(fdf, fdt, max_bytes);
679 else if (endswith(filename, ".xz"))
680 return decompress_stream_xz(fdf, fdt, max_bytes);
681 else
682 return -EPROTONOSUPPORT;
683 }