]> git.ipfire.org Git - thirdparty/systemd.git/blob - src/journal/compress.c
Merge pull request #2495 from heftig/master
[thirdparty/systemd.git] / src / journal / compress.c
1 /***
2 This file is part of systemd.
3
4 Copyright 2011 Lennart Poettering
5
6 systemd is free software; you can redistribute it and/or modify it
7 under the terms of the GNU Lesser General Public License as published by
8 the Free Software Foundation; either version 2.1 of the License, or
9 (at your option) any later version.
10
11 systemd is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public License
17 along with systemd; If not, see <http://www.gnu.org/licenses/>.
18 ***/
19
20 #include <stdlib.h>
21 #include <string.h>
22 #include <sys/mman.h>
23 #include <unistd.h>
24
25 #ifdef HAVE_XZ
26 #include <lzma.h>
27 #endif
28
29 #ifdef HAVE_LZ4
30 #include <lz4.h>
31 #include <lz4frame.h>
32 #endif
33
34 #include "alloc-util.h"
35 #include "compress.h"
36 #include "fd-util.h"
37 #include "io-util.h"
38 #include "journal-def.h"
39 #include "macro.h"
40 #include "sparse-endian.h"
41 #include "string-table.h"
42 #include "string-util.h"
43 #include "util.h"
44
45 #ifdef HAVE_LZ4
46 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t, LZ4F_freeCompressionContext);
47 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext);
48 #endif
49
50 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
51
52 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
53 [OBJECT_COMPRESSED_XZ] = "XZ",
54 [OBJECT_COMPRESSED_LZ4] = "LZ4",
55 };
56
57 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
58
59 int compress_blob_xz(const void *src, uint64_t src_size,
60 void *dst, size_t dst_alloc_size, size_t *dst_size) {
61 #ifdef HAVE_XZ
62 static const lzma_options_lzma opt = {
63 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
64 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
65 };
66 static const lzma_filter filters[] = {
67 { LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt },
68 { LZMA_VLI_UNKNOWN, NULL }
69 };
70 lzma_ret ret;
71 size_t out_pos = 0;
72
73 assert(src);
74 assert(src_size > 0);
75 assert(dst);
76 assert(dst_alloc_size > 0);
77 assert(dst_size);
78
79 /* Returns < 0 if we couldn't compress the data or the
80 * compressed result is longer than the original */
81
82 if (src_size < 80)
83 return -ENOBUFS;
84
85 ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
86 src, src_size, dst, &out_pos, dst_alloc_size);
87 if (ret != LZMA_OK)
88 return -ENOBUFS;
89
90 *dst_size = out_pos;
91 return 0;
92 #else
93 return -EPROTONOSUPPORT;
94 #endif
95 }
96
97 int compress_blob_lz4(const void *src, uint64_t src_size,
98 void *dst, size_t dst_alloc_size, size_t *dst_size) {
99 #ifdef HAVE_LZ4
100 int r;
101
102 assert(src);
103 assert(src_size > 0);
104 assert(dst);
105 assert(dst_alloc_size > 0);
106 assert(dst_size);
107
108 /* Returns < 0 if we couldn't compress the data or the
109 * compressed result is longer than the original */
110
111 if (src_size < 9)
112 return -ENOBUFS;
113
114 r = LZ4_compress_limitedOutput(src, dst + 8, src_size, (int) dst_alloc_size - 8);
115 if (r <= 0)
116 return -ENOBUFS;
117
118 *(le64_t*) dst = htole64(src_size);
119 *dst_size = r + 8;
120
121 return 0;
122 #else
123 return -EPROTONOSUPPORT;
124 #endif
125 }
126
127
128 int decompress_blob_xz(const void *src, uint64_t src_size,
129 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
130
131 #ifdef HAVE_XZ
132 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
133 lzma_ret ret;
134 size_t space;
135
136 assert(src);
137 assert(src_size > 0);
138 assert(dst);
139 assert(dst_alloc_size);
140 assert(dst_size);
141 assert(*dst_alloc_size == 0 || *dst);
142
143 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
144 if (ret != LZMA_OK)
145 return -ENOMEM;
146
147 space = MIN(src_size * 2, dst_max ?: (size_t) -1);
148 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
149 return -ENOMEM;
150
151 s.next_in = src;
152 s.avail_in = src_size;
153
154 s.next_out = *dst;
155 s.avail_out = space;
156
157 for (;;) {
158 size_t used;
159
160 ret = lzma_code(&s, LZMA_FINISH);
161
162 if (ret == LZMA_STREAM_END)
163 break;
164 else if (ret != LZMA_OK)
165 return -ENOMEM;
166
167 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
168 break;
169 else if (dst_max > 0 && space == dst_max)
170 return -ENOBUFS;
171
172 used = space - s.avail_out;
173 space = MIN(2 * space, dst_max ?: (size_t) -1);
174 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
175 return -ENOMEM;
176
177 s.avail_out = space - used;
178 s.next_out = *dst + used;
179 }
180
181 *dst_size = space - s.avail_out;
182 return 0;
183 #else
184 return -EPROTONOSUPPORT;
185 #endif
186 }
187
188 int decompress_blob_lz4(const void *src, uint64_t src_size,
189 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
190
191 #ifdef HAVE_LZ4
192 char* out;
193 int r, size; /* LZ4 uses int for size */
194
195 assert(src);
196 assert(src_size > 0);
197 assert(dst);
198 assert(dst_alloc_size);
199 assert(dst_size);
200 assert(*dst_alloc_size == 0 || *dst);
201
202 if (src_size <= 8)
203 return -EBADMSG;
204
205 size = le64toh( *(le64_t*)src );
206 if (size < 0 || (unsigned) size != le64toh(*(le64_t*)src))
207 return -EFBIG;
208 if ((size_t) size > *dst_alloc_size) {
209 out = realloc(*dst, size);
210 if (!out)
211 return -ENOMEM;
212 *dst = out;
213 *dst_alloc_size = size;
214 } else
215 out = *dst;
216
217 r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
218 if (r < 0 || r != size)
219 return -EBADMSG;
220
221 *dst_size = size;
222 return 0;
223 #else
224 return -EPROTONOSUPPORT;
225 #endif
226 }
227
228 int decompress_blob(int compression,
229 const void *src, uint64_t src_size,
230 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
231 if (compression == OBJECT_COMPRESSED_XZ)
232 return decompress_blob_xz(src, src_size,
233 dst, dst_alloc_size, dst_size, dst_max);
234 else if (compression == OBJECT_COMPRESSED_LZ4)
235 return decompress_blob_lz4(src, src_size,
236 dst, dst_alloc_size, dst_size, dst_max);
237 else
238 return -EBADMSG;
239 }
240
241
242 int decompress_startswith_xz(const void *src, uint64_t src_size,
243 void **buffer, size_t *buffer_size,
244 const void *prefix, size_t prefix_len,
245 uint8_t extra) {
246
247 #ifdef HAVE_XZ
248 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
249 lzma_ret ret;
250
251 /* Checks whether the decompressed blob starts with the
252 * mentioned prefix. The byte extra needs to follow the
253 * prefix */
254
255 assert(src);
256 assert(src_size > 0);
257 assert(buffer);
258 assert(buffer_size);
259 assert(prefix);
260 assert(*buffer_size == 0 || *buffer);
261
262 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
263 if (ret != LZMA_OK)
264 return -EBADMSG;
265
266 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
267 return -ENOMEM;
268
269 s.next_in = src;
270 s.avail_in = src_size;
271
272 s.next_out = *buffer;
273 s.avail_out = *buffer_size;
274
275 for (;;) {
276 ret = lzma_code(&s, LZMA_FINISH);
277
278 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
279 return -EBADMSG;
280
281 if (*buffer_size - s.avail_out >= prefix_len + 1)
282 return memcmp(*buffer, prefix, prefix_len) == 0 &&
283 ((const uint8_t*) *buffer)[prefix_len] == extra;
284
285 if (ret == LZMA_STREAM_END)
286 return 0;
287
288 s.avail_out += *buffer_size;
289
290 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
291 return -ENOMEM;
292
293 s.next_out = *buffer + *buffer_size - s.avail_out;
294 }
295
296 #else
297 return -EPROTONOSUPPORT;
298 #endif
299 }
300
301 int decompress_startswith_lz4(const void *src, uint64_t src_size,
302 void **buffer, size_t *buffer_size,
303 const void *prefix, size_t prefix_len,
304 uint8_t extra) {
305 #ifdef HAVE_LZ4
306 /* Checks whether the decompressed blob starts with the
307 * mentioned prefix. The byte extra needs to follow the
308 * prefix */
309
310 int r;
311 size_t size;
312
313 assert(src);
314 assert(src_size > 0);
315 assert(buffer);
316 assert(buffer_size);
317 assert(prefix);
318 assert(*buffer_size == 0 || *buffer);
319
320 if (src_size <= 8)
321 return -EBADMSG;
322
323 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
324 return -ENOMEM;
325
326 r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
327 prefix_len + 1, *buffer_size);
328 if (r >= 0)
329 size = (unsigned) r;
330 else {
331 /* lz4 always tries to decode full "sequence", so in
332 * pathological cases might need to decompress the
333 * full field. */
334 r = decompress_blob_lz4(src, src_size, buffer, buffer_size, &size, 0);
335 if (r < 0)
336 return r;
337 }
338
339 if (size >= prefix_len + 1)
340 return memcmp(*buffer, prefix, prefix_len) == 0 &&
341 ((const uint8_t*) *buffer)[prefix_len] == extra;
342 else
343 return 0;
344
345 #else
346 return -EPROTONOSUPPORT;
347 #endif
348 }
349
350 int decompress_startswith(int compression,
351 const void *src, uint64_t src_size,
352 void **buffer, size_t *buffer_size,
353 const void *prefix, size_t prefix_len,
354 uint8_t extra) {
355 if (compression == OBJECT_COMPRESSED_XZ)
356 return decompress_startswith_xz(src, src_size,
357 buffer, buffer_size,
358 prefix, prefix_len,
359 extra);
360 else if (compression == OBJECT_COMPRESSED_LZ4)
361 return decompress_startswith_lz4(src, src_size,
362 buffer, buffer_size,
363 prefix, prefix_len,
364 extra);
365 else
366 return -EBADMSG;
367 }
368
369 int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
370 #ifdef HAVE_XZ
371 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
372 lzma_ret ret;
373 uint8_t buf[BUFSIZ], out[BUFSIZ];
374 lzma_action action = LZMA_RUN;
375
376 assert(fdf >= 0);
377 assert(fdt >= 0);
378
379 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
380 if (ret != LZMA_OK) {
381 log_error("Failed to initialize XZ encoder: code %u", ret);
382 return -EINVAL;
383 }
384
385 for (;;) {
386 if (s.avail_in == 0 && action == LZMA_RUN) {
387 size_t m = sizeof(buf);
388 ssize_t n;
389
390 if (max_bytes != (uint64_t) -1 && (uint64_t) m > max_bytes)
391 m = (size_t) max_bytes;
392
393 n = read(fdf, buf, m);
394 if (n < 0)
395 return -errno;
396 if (n == 0)
397 action = LZMA_FINISH;
398 else {
399 s.next_in = buf;
400 s.avail_in = n;
401
402 if (max_bytes != (uint64_t) -1) {
403 assert(max_bytes >= (uint64_t) n);
404 max_bytes -= n;
405 }
406 }
407 }
408
409 if (s.avail_out == 0) {
410 s.next_out = out;
411 s.avail_out = sizeof(out);
412 }
413
414 ret = lzma_code(&s, action);
415 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
416 log_error("Compression failed: code %u", ret);
417 return -EBADMSG;
418 }
419
420 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
421 ssize_t n, k;
422
423 n = sizeof(out) - s.avail_out;
424
425 k = loop_write(fdt, out, n, false);
426 if (k < 0)
427 return k;
428
429 if (ret == LZMA_STREAM_END) {
430 log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
431 s.total_in, s.total_out,
432 (double) s.total_out / s.total_in * 100);
433
434 return 0;
435 }
436 }
437 }
438 #else
439 return -EPROTONOSUPPORT;
440 #endif
441 }
442
443 #define LZ4_BUFSIZE (512*1024u)
444
445 int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
446
447 #ifdef HAVE_LZ4
448 LZ4F_errorCode_t c;
449 _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
450 _cleanup_free_ char *buf = NULL;
451 char *src = NULL;
452 size_t size, n, total_in = 0, total_out, offset = 0, frame_size;
453 struct stat st;
454 int r;
455 static const LZ4F_compressOptions_t options = {
456 .stableSrc = 1,
457 };
458 static const LZ4F_preferences_t preferences = {
459 .frameInfo.blockSizeID = 5,
460 };
461
462 c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
463 if (LZ4F_isError(c))
464 return -ENOMEM;
465
466 if (fstat(fdf, &st) < 0)
467 return log_debug_errno(errno, "fstat() failed: %m");
468
469 frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
470 size = frame_size + 64*1024; /* add some space for header and trailer */
471 buf = malloc(size);
472 if (!buf)
473 return -ENOMEM;
474
475 n = offset = total_out = LZ4F_compressBegin(ctx, buf, size, &preferences);
476 if (LZ4F_isError(n))
477 return -EINVAL;
478
479 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0);
480 if (src == MAP_FAILED)
481 return -errno;
482
483 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size, n);
484
485 while (total_in < (size_t) st.st_size) {
486 ssize_t k;
487
488 k = MIN(LZ4_BUFSIZE, st.st_size - total_in);
489 n = LZ4F_compressUpdate(ctx, buf + offset, size - offset,
490 src + total_in, k, &options);
491 if (LZ4F_isError(n)) {
492 r = -ENOTRECOVERABLE;
493 goto cleanup;
494 }
495
496 total_in += k;
497 offset += n;
498 total_out += n;
499
500 if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) {
501 log_debug("Compressed stream longer than %zd bytes", max_bytes);
502 return -EFBIG;
503 }
504
505 if (size - offset < frame_size + 4) {
506 k = loop_write(fdt, buf, offset, false);
507 if (k < 0) {
508 r = k;
509 goto cleanup;
510 }
511 offset = 0;
512 }
513 }
514
515 n = LZ4F_compressEnd(ctx, buf + offset, size - offset, &options);
516 if (LZ4F_isError(n)) {
517 r = -ENOTRECOVERABLE;
518 goto cleanup;
519 }
520
521 offset += n;
522 total_out += n;
523 r = loop_write(fdt, buf, offset, false);
524 if (r < 0)
525 goto cleanup;
526
527 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
528 total_in, total_out,
529 (double) total_out / total_in * 100);
530 cleanup:
531 munmap(src, st.st_size);
532 return r;
533 #else
534 return -EPROTONOSUPPORT;
535 #endif
536 }
537
538 int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
539
540 #ifdef HAVE_XZ
541 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
542 lzma_ret ret;
543
544 uint8_t buf[BUFSIZ], out[BUFSIZ];
545 lzma_action action = LZMA_RUN;
546
547 assert(fdf >= 0);
548 assert(fdt >= 0);
549
550 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
551 if (ret != LZMA_OK) {
552 log_debug("Failed to initialize XZ decoder: code %u", ret);
553 return -ENOMEM;
554 }
555
556 for (;;) {
557 if (s.avail_in == 0 && action == LZMA_RUN) {
558 ssize_t n;
559
560 n = read(fdf, buf, sizeof(buf));
561 if (n < 0)
562 return -errno;
563 if (n == 0)
564 action = LZMA_FINISH;
565 else {
566 s.next_in = buf;
567 s.avail_in = n;
568 }
569 }
570
571 if (s.avail_out == 0) {
572 s.next_out = out;
573 s.avail_out = sizeof(out);
574 }
575
576 ret = lzma_code(&s, action);
577 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
578 log_debug("Decompression failed: code %u", ret);
579 return -EBADMSG;
580 }
581
582 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
583 ssize_t n, k;
584
585 n = sizeof(out) - s.avail_out;
586
587 if (max_bytes != (uint64_t) -1) {
588 if (max_bytes < (uint64_t) n)
589 return -EFBIG;
590
591 max_bytes -= n;
592 }
593
594 k = loop_write(fdt, out, n, false);
595 if (k < 0)
596 return k;
597
598 if (ret == LZMA_STREAM_END) {
599 log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
600 s.total_in, s.total_out,
601 (double) s.total_out / s.total_in * 100);
602
603 return 0;
604 }
605 }
606 }
607 #else
608 log_debug("Cannot decompress file. Compiled without XZ support.");
609 return -EPROTONOSUPPORT;
610 #endif
611 }
612
613 int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
614 #ifdef HAVE_LZ4
615 size_t c;
616 _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
617 _cleanup_free_ char *buf = NULL;
618 char *src;
619 struct stat st;
620 int r = 0;
621 size_t total_in = 0, total_out = 0;
622
623 c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
624 if (LZ4F_isError(c))
625 return -ENOMEM;
626
627 if (fstat(in, &st) < 0)
628 return log_debug_errno(errno, "fstat() failed: %m");
629
630 buf = malloc(LZ4_BUFSIZE);
631 if (!buf)
632 return -ENOMEM;
633
634 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in, 0);
635 if (src == MAP_FAILED)
636 return -errno;
637
638 while (total_in < (size_t) st.st_size) {
639 size_t produced = LZ4_BUFSIZE;
640 size_t used = st.st_size - total_in;
641
642 c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
643 if (LZ4F_isError(c)) {
644 r = -EBADMSG;
645 goto cleanup;
646 }
647
648 total_in += used;
649 total_out += produced;
650
651 if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) {
652 log_debug("Decompressed stream longer than %zd bytes", max_bytes);
653 r = -EFBIG;
654 goto cleanup;
655 }
656
657 r = loop_write(out, buf, produced, false);
658 if (r < 0)
659 goto cleanup;
660 }
661
662 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
663 total_in, total_out,
664 (double) total_out / total_in * 100);
665 cleanup:
666 munmap(src, st.st_size);
667 return r;
668 #else
669 log_debug("Cannot decompress file. Compiled without LZ4 support.");
670 return -EPROTONOSUPPORT;
671 #endif
672 }
673
674 int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
675
676 if (endswith(filename, ".lz4"))
677 return decompress_stream_lz4(fdf, fdt, max_bytes);
678 else if (endswith(filename, ".xz"))
679 return decompress_stream_xz(fdf, fdt, max_bytes);
680 else
681 return -EPROTONOSUPPORT;
682 }