]> git.ipfire.org Git - thirdparty/systemd.git/blob - src/journal/compress.c
Merge pull request #2391 from keszybz/coverity
[thirdparty/systemd.git] / src / journal / compress.c
1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
2
3 /***
4 This file is part of systemd.
5
6 Copyright 2011 Lennart Poettering
7
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
12
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
17
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
20 ***/
21
22 #include <stdlib.h>
23 #include <string.h>
24 #include <sys/mman.h>
25 #include <unistd.h>
26
27 #ifdef HAVE_XZ
28 #include <lzma.h>
29 #endif
30
31 #ifdef HAVE_LZ4
32 #include <lz4.h>
33 #include <lz4frame.h>
34 #endif
35
36 #include "alloc-util.h"
37 #include "compress.h"
38 #include "fd-util.h"
39 #include "io-util.h"
40 #include "journal-def.h"
41 #include "macro.h"
42 #include "sparse-endian.h"
43 #include "string-table.h"
44 #include "string-util.h"
45 #include "util.h"
46
47 #ifdef HAVE_LZ4
48 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t, LZ4F_freeCompressionContext);
49 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext);
50 #endif
51
52 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
53
54 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
55 [OBJECT_COMPRESSED_XZ] = "XZ",
56 [OBJECT_COMPRESSED_LZ4] = "LZ4",
57 };
58
59 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
60
61 int compress_blob_xz(const void *src, uint64_t src_size,
62 void *dst, size_t dst_alloc_size, size_t *dst_size) {
63 #ifdef HAVE_XZ
64 static const lzma_options_lzma opt = {
65 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
66 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
67 };
68 static const lzma_filter filters[] = {
69 { LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt },
70 { LZMA_VLI_UNKNOWN, NULL }
71 };
72 lzma_ret ret;
73 size_t out_pos = 0;
74
75 assert(src);
76 assert(src_size > 0);
77 assert(dst);
78 assert(dst_alloc_size > 0);
79 assert(dst_size);
80
81 /* Returns < 0 if we couldn't compress the data or the
82 * compressed result is longer than the original */
83
84 if (src_size < 80)
85 return -ENOBUFS;
86
87 ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
88 src, src_size, dst, &out_pos, dst_alloc_size);
89 if (ret != LZMA_OK)
90 return -ENOBUFS;
91
92 *dst_size = out_pos;
93 return 0;
94 #else
95 return -EPROTONOSUPPORT;
96 #endif
97 }
98
99 int compress_blob_lz4(const void *src, uint64_t src_size,
100 void *dst, size_t dst_alloc_size, size_t *dst_size) {
101 #ifdef HAVE_LZ4
102 int r;
103
104 assert(src);
105 assert(src_size > 0);
106 assert(dst);
107 assert(dst_alloc_size > 0);
108 assert(dst_size);
109
110 /* Returns < 0 if we couldn't compress the data or the
111 * compressed result is longer than the original */
112
113 if (src_size < 9)
114 return -ENOBUFS;
115
116 r = LZ4_compress_limitedOutput(src, dst + 8, src_size, (int) dst_alloc_size - 8);
117 if (r <= 0)
118 return -ENOBUFS;
119
120 *(le64_t*) dst = htole64(src_size);
121 *dst_size = r + 8;
122
123 return 0;
124 #else
125 return -EPROTONOSUPPORT;
126 #endif
127 }
128
129
130 int decompress_blob_xz(const void *src, uint64_t src_size,
131 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
132
133 #ifdef HAVE_XZ
134 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
135 lzma_ret ret;
136 size_t space;
137
138 assert(src);
139 assert(src_size > 0);
140 assert(dst);
141 assert(dst_alloc_size);
142 assert(dst_size);
143 assert(*dst_alloc_size == 0 || *dst);
144
145 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
146 if (ret != LZMA_OK)
147 return -ENOMEM;
148
149 space = MIN(src_size * 2, dst_max ?: (size_t) -1);
150 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
151 return -ENOMEM;
152
153 s.next_in = src;
154 s.avail_in = src_size;
155
156 s.next_out = *dst;
157 s.avail_out = space;
158
159 for (;;) {
160 size_t used;
161
162 ret = lzma_code(&s, LZMA_FINISH);
163
164 if (ret == LZMA_STREAM_END)
165 break;
166 else if (ret != LZMA_OK)
167 return -ENOMEM;
168
169 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
170 break;
171 else if (dst_max > 0 && space == dst_max)
172 return -ENOBUFS;
173
174 used = space - s.avail_out;
175 space = MIN(2 * space, dst_max ?: (size_t) -1);
176 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
177 return -ENOMEM;
178
179 s.avail_out = space - used;
180 s.next_out = *dst + used;
181 }
182
183 *dst_size = space - s.avail_out;
184 return 0;
185 #else
186 return -EPROTONOSUPPORT;
187 #endif
188 }
189
190 int decompress_blob_lz4(const void *src, uint64_t src_size,
191 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
192
193 #ifdef HAVE_LZ4
194 char* out;
195 int r, size; /* LZ4 uses int for size */
196
197 assert(src);
198 assert(src_size > 0);
199 assert(dst);
200 assert(dst_alloc_size);
201 assert(dst_size);
202 assert(*dst_alloc_size == 0 || *dst);
203
204 if (src_size <= 8)
205 return -EBADMSG;
206
207 size = le64toh( *(le64_t*)src );
208 if (size < 0 || (unsigned) size != le64toh(*(le64_t*)src))
209 return -EFBIG;
210 if ((size_t) size > *dst_alloc_size) {
211 out = realloc(*dst, size);
212 if (!out)
213 return -ENOMEM;
214 *dst = out;
215 *dst_alloc_size = size;
216 } else
217 out = *dst;
218
219 r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
220 if (r < 0 || r != size)
221 return -EBADMSG;
222
223 *dst_size = size;
224 return 0;
225 #else
226 return -EPROTONOSUPPORT;
227 #endif
228 }
229
230 int decompress_blob(int compression,
231 const void *src, uint64_t src_size,
232 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
233 if (compression == OBJECT_COMPRESSED_XZ)
234 return decompress_blob_xz(src, src_size,
235 dst, dst_alloc_size, dst_size, dst_max);
236 else if (compression == OBJECT_COMPRESSED_LZ4)
237 return decompress_blob_lz4(src, src_size,
238 dst, dst_alloc_size, dst_size, dst_max);
239 else
240 return -EBADMSG;
241 }
242
243
244 int decompress_startswith_xz(const void *src, uint64_t src_size,
245 void **buffer, size_t *buffer_size,
246 const void *prefix, size_t prefix_len,
247 uint8_t extra) {
248
249 #ifdef HAVE_XZ
250 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
251 lzma_ret ret;
252
253 /* Checks whether the decompressed blob starts with the
254 * mentioned prefix. The byte extra needs to follow the
255 * prefix */
256
257 assert(src);
258 assert(src_size > 0);
259 assert(buffer);
260 assert(buffer_size);
261 assert(prefix);
262 assert(*buffer_size == 0 || *buffer);
263
264 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
265 if (ret != LZMA_OK)
266 return -EBADMSG;
267
268 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
269 return -ENOMEM;
270
271 s.next_in = src;
272 s.avail_in = src_size;
273
274 s.next_out = *buffer;
275 s.avail_out = *buffer_size;
276
277 for (;;) {
278 ret = lzma_code(&s, LZMA_FINISH);
279
280 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
281 return -EBADMSG;
282
283 if (*buffer_size - s.avail_out >= prefix_len + 1)
284 return memcmp(*buffer, prefix, prefix_len) == 0 &&
285 ((const uint8_t*) *buffer)[prefix_len] == extra;
286
287 if (ret == LZMA_STREAM_END)
288 return 0;
289
290 s.avail_out += *buffer_size;
291
292 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
293 return -ENOMEM;
294
295 s.next_out = *buffer + *buffer_size - s.avail_out;
296 }
297
298 #else
299 return -EPROTONOSUPPORT;
300 #endif
301 }
302
303 int decompress_startswith_lz4(const void *src, uint64_t src_size,
304 void **buffer, size_t *buffer_size,
305 const void *prefix, size_t prefix_len,
306 uint8_t extra) {
307 #ifdef HAVE_LZ4
308 /* Checks whether the decompressed blob starts with the
309 * mentioned prefix. The byte extra needs to follow the
310 * prefix */
311
312 int r;
313 size_t size;
314
315 assert(src);
316 assert(src_size > 0);
317 assert(buffer);
318 assert(buffer_size);
319 assert(prefix);
320 assert(*buffer_size == 0 || *buffer);
321
322 if (src_size <= 8)
323 return -EBADMSG;
324
325 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
326 return -ENOMEM;
327
328 r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
329 prefix_len + 1, *buffer_size);
330 if (r >= 0)
331 size = (unsigned) r;
332 else {
333 /* lz4 always tries to decode full "sequence", so in
334 * pathological cases might need to decompress the
335 * full field. */
336 r = decompress_blob_lz4(src, src_size, buffer, buffer_size, &size, 0);
337 if (r < 0)
338 return r;
339 }
340
341 if (size >= prefix_len + 1)
342 return memcmp(*buffer, prefix, prefix_len) == 0 &&
343 ((const uint8_t*) *buffer)[prefix_len] == extra;
344 else
345 return 0;
346
347 #else
348 return -EPROTONOSUPPORT;
349 #endif
350 }
351
352 int decompress_startswith(int compression,
353 const void *src, uint64_t src_size,
354 void **buffer, size_t *buffer_size,
355 const void *prefix, size_t prefix_len,
356 uint8_t extra) {
357 if (compression == OBJECT_COMPRESSED_XZ)
358 return decompress_startswith_xz(src, src_size,
359 buffer, buffer_size,
360 prefix, prefix_len,
361 extra);
362 else if (compression == OBJECT_COMPRESSED_LZ4)
363 return decompress_startswith_lz4(src, src_size,
364 buffer, buffer_size,
365 prefix, prefix_len,
366 extra);
367 else
368 return -EBADMSG;
369 }
370
371 int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
372 #ifdef HAVE_XZ
373 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
374 lzma_ret ret;
375 uint8_t buf[BUFSIZ], out[BUFSIZ];
376 lzma_action action = LZMA_RUN;
377
378 assert(fdf >= 0);
379 assert(fdt >= 0);
380
381 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
382 if (ret != LZMA_OK) {
383 log_error("Failed to initialize XZ encoder: code %u", ret);
384 return -EINVAL;
385 }
386
387 for (;;) {
388 if (s.avail_in == 0 && action == LZMA_RUN) {
389 size_t m = sizeof(buf);
390 ssize_t n;
391
392 if (max_bytes != (uint64_t) -1 && (uint64_t) m > max_bytes)
393 m = (size_t) max_bytes;
394
395 n = read(fdf, buf, m);
396 if (n < 0)
397 return -errno;
398 if (n == 0)
399 action = LZMA_FINISH;
400 else {
401 s.next_in = buf;
402 s.avail_in = n;
403
404 if (max_bytes != (uint64_t) -1) {
405 assert(max_bytes >= (uint64_t) n);
406 max_bytes -= n;
407 }
408 }
409 }
410
411 if (s.avail_out == 0) {
412 s.next_out = out;
413 s.avail_out = sizeof(out);
414 }
415
416 ret = lzma_code(&s, action);
417 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
418 log_error("Compression failed: code %u", ret);
419 return -EBADMSG;
420 }
421
422 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
423 ssize_t n, k;
424
425 n = sizeof(out) - s.avail_out;
426
427 k = loop_write(fdt, out, n, false);
428 if (k < 0)
429 return k;
430
431 if (ret == LZMA_STREAM_END) {
432 log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
433 s.total_in, s.total_out,
434 (double) s.total_out / s.total_in * 100);
435
436 return 0;
437 }
438 }
439 }
440 #else
441 return -EPROTONOSUPPORT;
442 #endif
443 }
444
445 #define LZ4_BUFSIZE (512*1024u)
446
447 int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
448
449 #ifdef HAVE_LZ4
450 LZ4F_errorCode_t c;
451 _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
452 _cleanup_free_ char *buf = NULL;
453 char *src = NULL;
454 size_t size, n, total_in = 0, total_out, offset = 0, frame_size;
455 struct stat st;
456 int r;
457 static const LZ4F_compressOptions_t options = {
458 .stableSrc = 1,
459 };
460 static const LZ4F_preferences_t preferences = {
461 .frameInfo.blockSizeID = 5,
462 };
463
464 c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
465 if (LZ4F_isError(c))
466 return -ENOMEM;
467
468 if (fstat(fdf, &st) < 0)
469 return log_debug_errno(errno, "fstat() failed: %m");
470
471 frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
472 size = frame_size + 64*1024; /* add some space for header and trailer */
473 buf = malloc(size);
474 if (!buf)
475 return -ENOMEM;
476
477 n = offset = total_out = LZ4F_compressBegin(ctx, buf, size, &preferences);
478 if (LZ4F_isError(n))
479 return -EINVAL;
480
481 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0);
482 if (src == MAP_FAILED)
483 return -errno;
484
485 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size, n);
486
487 while (total_in < (size_t) st.st_size) {
488 ssize_t k;
489
490 k = MIN(LZ4_BUFSIZE, st.st_size - total_in);
491 n = LZ4F_compressUpdate(ctx, buf + offset, size - offset,
492 src + total_in, k, &options);
493 if (LZ4F_isError(n)) {
494 r = -ENOTRECOVERABLE;
495 goto cleanup;
496 }
497
498 total_in += k;
499 offset += n;
500 total_out += n;
501
502 if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) {
503 log_debug("Compressed stream longer than %zd bytes", max_bytes);
504 return -EFBIG;
505 }
506
507 if (size - offset < frame_size + 4) {
508 k = loop_write(fdt, buf, offset, false);
509 if (k < 0) {
510 r = k;
511 goto cleanup;
512 }
513 offset = 0;
514 }
515 }
516
517 n = LZ4F_compressEnd(ctx, buf + offset, size - offset, &options);
518 if (LZ4F_isError(n)) {
519 r = -ENOTRECOVERABLE;
520 goto cleanup;
521 }
522
523 offset += n;
524 total_out += n;
525 r = loop_write(fdt, buf, offset, false);
526 if (r < 0)
527 goto cleanup;
528
529 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
530 total_in, total_out,
531 (double) total_out / total_in * 100);
532 cleanup:
533 munmap(src, st.st_size);
534 return r;
535 #else
536 return -EPROTONOSUPPORT;
537 #endif
538 }
539
540 int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
541
542 #ifdef HAVE_XZ
543 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
544 lzma_ret ret;
545
546 uint8_t buf[BUFSIZ], out[BUFSIZ];
547 lzma_action action = LZMA_RUN;
548
549 assert(fdf >= 0);
550 assert(fdt >= 0);
551
552 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
553 if (ret != LZMA_OK) {
554 log_debug("Failed to initialize XZ decoder: code %u", ret);
555 return -ENOMEM;
556 }
557
558 for (;;) {
559 if (s.avail_in == 0 && action == LZMA_RUN) {
560 ssize_t n;
561
562 n = read(fdf, buf, sizeof(buf));
563 if (n < 0)
564 return -errno;
565 if (n == 0)
566 action = LZMA_FINISH;
567 else {
568 s.next_in = buf;
569 s.avail_in = n;
570 }
571 }
572
573 if (s.avail_out == 0) {
574 s.next_out = out;
575 s.avail_out = sizeof(out);
576 }
577
578 ret = lzma_code(&s, action);
579 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
580 log_debug("Decompression failed: code %u", ret);
581 return -EBADMSG;
582 }
583
584 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
585 ssize_t n, k;
586
587 n = sizeof(out) - s.avail_out;
588
589 if (max_bytes != (uint64_t) -1) {
590 if (max_bytes < (uint64_t) n)
591 return -EFBIG;
592
593 max_bytes -= n;
594 }
595
596 k = loop_write(fdt, out, n, false);
597 if (k < 0)
598 return k;
599
600 if (ret == LZMA_STREAM_END) {
601 log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
602 s.total_in, s.total_out,
603 (double) s.total_out / s.total_in * 100);
604
605 return 0;
606 }
607 }
608 }
609 #else
610 log_debug("Cannot decompress file. Compiled without XZ support.");
611 return -EPROTONOSUPPORT;
612 #endif
613 }
614
615 int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
616 #ifdef HAVE_LZ4
617 size_t c;
618 _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
619 _cleanup_free_ char *buf = NULL;
620 char *src;
621 struct stat st;
622 int r = 0;
623 size_t total_in = 0, total_out = 0;
624
625 c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
626 if (LZ4F_isError(c))
627 return -ENOMEM;
628
629 if (fstat(in, &st) < 0)
630 return log_debug_errno(errno, "fstat() failed: %m");
631
632 buf = malloc(LZ4_BUFSIZE);
633 if (!buf)
634 return -ENOMEM;
635
636 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in, 0);
637 if (src == MAP_FAILED)
638 return -errno;
639
640 while (total_in < (size_t) st.st_size) {
641 size_t produced = LZ4_BUFSIZE;
642 size_t used = st.st_size - total_in;
643
644 c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
645 if (LZ4F_isError(c)) {
646 r = -EBADMSG;
647 goto cleanup;
648 }
649
650 total_in += used;
651 total_out += produced;
652
653 if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) {
654 log_debug("Decompressed stream longer than %zd bytes", max_bytes);
655 r = -EFBIG;
656 goto cleanup;
657 }
658
659 r = loop_write(out, buf, produced, false);
660 if (r < 0)
661 goto cleanup;
662 }
663
664 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
665 total_in, total_out,
666 (double) total_out / total_in * 100);
667 cleanup:
668 munmap(src, st.st_size);
669 return r;
670 #else
671 log_debug("Cannot decompress file. Compiled without LZ4 support.");
672 return -EPROTONOSUPPORT;
673 #endif
674 }
675
676 int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
677
678 if (endswith(filename, ".lz4"))
679 return decompress_stream_lz4(fdf, fdt, max_bytes);
680 else if (endswith(filename, ".xz"))
681 return decompress_stream_xz(fdf, fdt, max_bytes);
682 else
683 return -EPROTONOSUPPORT;
684 }