]> git.ipfire.org Git - thirdparty/systemd.git/blob - src/journal/compress.c
journal: avoid undefined behaviour in float division by 0.0
[thirdparty/systemd.git] / src / journal / compress.c
1 /* SPDX-License-Identifier: LGPL-2.1+ */
2 /***
3 This file is part of systemd.
4
5 Copyright 2011 Lennart Poettering
6
7 systemd is free software; you can redistribute it and/or modify it
8 under the terms of the GNU Lesser General Public License as published by
9 the Free Software Foundation; either version 2.1 of the License, or
10 (at your option) any later version.
11
12 systemd is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
16
17 You should have received a copy of the GNU Lesser General Public License
18 along with systemd; If not, see <http://www.gnu.org/licenses/>.
19 ***/
20
21 #include <inttypes.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <sys/mman.h>
25 #include <unistd.h>
26
27 #if HAVE_XZ
28 #include <lzma.h>
29 #endif
30
31 #if HAVE_LZ4
32 #include <lz4.h>
33 #include <lz4frame.h>
34 #endif
35
36 #include "alloc-util.h"
37 #include "compress.h"
38 #include "fd-util.h"
39 #include "io-util.h"
40 #include "journal-def.h"
41 #include "macro.h"
42 #include "sparse-endian.h"
43 #include "string-table.h"
44 #include "string-util.h"
45 #include "util.h"
46
47 #if HAVE_LZ4
48 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t, LZ4F_freeCompressionContext);
49 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext);
50 #endif
51
52 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
53
54 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
55 [OBJECT_COMPRESSED_XZ] = "XZ",
56 [OBJECT_COMPRESSED_LZ4] = "LZ4",
57 };
58
59 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
60
61 int compress_blob_xz(const void *src, uint64_t src_size,
62 void *dst, size_t dst_alloc_size, size_t *dst_size) {
63 #if HAVE_XZ
64 static const lzma_options_lzma opt = {
65 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
66 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
67 };
68 static const lzma_filter filters[] = {
69 { LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt },
70 { LZMA_VLI_UNKNOWN, NULL }
71 };
72 lzma_ret ret;
73 size_t out_pos = 0;
74
75 assert(src);
76 assert(src_size > 0);
77 assert(dst);
78 assert(dst_alloc_size > 0);
79 assert(dst_size);
80
81 /* Returns < 0 if we couldn't compress the data or the
82 * compressed result is longer than the original */
83
84 if (src_size < 80)
85 return -ENOBUFS;
86
87 ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
88 src, src_size, dst, &out_pos, dst_alloc_size);
89 if (ret != LZMA_OK)
90 return -ENOBUFS;
91
92 *dst_size = out_pos;
93 return 0;
94 #else
95 return -EPROTONOSUPPORT;
96 #endif
97 }
98
99 int compress_blob_lz4(const void *src, uint64_t src_size,
100 void *dst, size_t dst_alloc_size, size_t *dst_size) {
101 #if HAVE_LZ4
102 int r;
103
104 assert(src);
105 assert(src_size > 0);
106 assert(dst);
107 assert(dst_alloc_size > 0);
108 assert(dst_size);
109
110 /* Returns < 0 if we couldn't compress the data or the
111 * compressed result is longer than the original */
112
113 if (src_size < 9)
114 return -ENOBUFS;
115
116 #if LZ4_VERSION_NUMBER >= 10700
117 r = LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
118 #else
119 r = LZ4_compress_limitedOutput(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
120 #endif
121 if (r <= 0)
122 return -ENOBUFS;
123
124 *(le64_t*) dst = htole64(src_size);
125 *dst_size = r + 8;
126
127 return 0;
128 #else
129 return -EPROTONOSUPPORT;
130 #endif
131 }
132
133
134 int decompress_blob_xz(const void *src, uint64_t src_size,
135 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
136
137 #if HAVE_XZ
138 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
139 lzma_ret ret;
140 size_t space;
141
142 assert(src);
143 assert(src_size > 0);
144 assert(dst);
145 assert(dst_alloc_size);
146 assert(dst_size);
147 assert(*dst_alloc_size == 0 || *dst);
148
149 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
150 if (ret != LZMA_OK)
151 return -ENOMEM;
152
153 space = MIN(src_size * 2, dst_max ?: (size_t) -1);
154 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
155 return -ENOMEM;
156
157 s.next_in = src;
158 s.avail_in = src_size;
159
160 s.next_out = *dst;
161 s.avail_out = space;
162
163 for (;;) {
164 size_t used;
165
166 ret = lzma_code(&s, LZMA_FINISH);
167
168 if (ret == LZMA_STREAM_END)
169 break;
170 else if (ret != LZMA_OK)
171 return -ENOMEM;
172
173 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
174 break;
175 else if (dst_max > 0 && space == dst_max)
176 return -ENOBUFS;
177
178 used = space - s.avail_out;
179 space = MIN(2 * space, dst_max ?: (size_t) -1);
180 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
181 return -ENOMEM;
182
183 s.avail_out = space - used;
184 s.next_out = *(uint8_t**)dst + used;
185 }
186
187 *dst_size = space - s.avail_out;
188 return 0;
189 #else
190 return -EPROTONOSUPPORT;
191 #endif
192 }
193
194 int decompress_blob_lz4(const void *src, uint64_t src_size,
195 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
196
197 #if HAVE_LZ4
198 char* out;
199 int r, size; /* LZ4 uses int for size */
200
201 assert(src);
202 assert(src_size > 0);
203 assert(dst);
204 assert(dst_alloc_size);
205 assert(dst_size);
206 assert(*dst_alloc_size == 0 || *dst);
207
208 if (src_size <= 8)
209 return -EBADMSG;
210
211 size = le64toh( *(le64_t*)src );
212 if (size < 0 || (unsigned) size != le64toh(*(le64_t*)src))
213 return -EFBIG;
214 if ((size_t) size > *dst_alloc_size) {
215 out = realloc(*dst, size);
216 if (!out)
217 return -ENOMEM;
218 *dst = out;
219 *dst_alloc_size = size;
220 } else
221 out = *dst;
222
223 r = LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size);
224 if (r < 0 || r != size)
225 return -EBADMSG;
226
227 *dst_size = size;
228 return 0;
229 #else
230 return -EPROTONOSUPPORT;
231 #endif
232 }
233
234 int decompress_blob(int compression,
235 const void *src, uint64_t src_size,
236 void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
237 if (compression == OBJECT_COMPRESSED_XZ)
238 return decompress_blob_xz(src, src_size,
239 dst, dst_alloc_size, dst_size, dst_max);
240 else if (compression == OBJECT_COMPRESSED_LZ4)
241 return decompress_blob_lz4(src, src_size,
242 dst, dst_alloc_size, dst_size, dst_max);
243 else
244 return -EBADMSG;
245 }
246
247
248 int decompress_startswith_xz(const void *src, uint64_t src_size,
249 void **buffer, size_t *buffer_size,
250 const void *prefix, size_t prefix_len,
251 uint8_t extra) {
252
253 #if HAVE_XZ
254 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
255 lzma_ret ret;
256
257 /* Checks whether the decompressed blob starts with the
258 * mentioned prefix. The byte extra needs to follow the
259 * prefix */
260
261 assert(src);
262 assert(src_size > 0);
263 assert(buffer);
264 assert(buffer_size);
265 assert(prefix);
266 assert(*buffer_size == 0 || *buffer);
267
268 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
269 if (ret != LZMA_OK)
270 return -EBADMSG;
271
272 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
273 return -ENOMEM;
274
275 s.next_in = src;
276 s.avail_in = src_size;
277
278 s.next_out = *buffer;
279 s.avail_out = *buffer_size;
280
281 for (;;) {
282 ret = lzma_code(&s, LZMA_FINISH);
283
284 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
285 return -EBADMSG;
286
287 if (*buffer_size - s.avail_out >= prefix_len + 1)
288 return memcmp(*buffer, prefix, prefix_len) == 0 &&
289 ((const uint8_t*) *buffer)[prefix_len] == extra;
290
291 if (ret == LZMA_STREAM_END)
292 return 0;
293
294 s.avail_out += *buffer_size;
295
296 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
297 return -ENOMEM;
298
299 s.next_out = *(uint8_t**)buffer + *buffer_size - s.avail_out;
300 }
301
302 #else
303 return -EPROTONOSUPPORT;
304 #endif
305 }
306
307 int decompress_startswith_lz4(const void *src, uint64_t src_size,
308 void **buffer, size_t *buffer_size,
309 const void *prefix, size_t prefix_len,
310 uint8_t extra) {
311 #if HAVE_LZ4
312 /* Checks whether the decompressed blob starts with the
313 * mentioned prefix. The byte extra needs to follow the
314 * prefix */
315
316 int r;
317 size_t size;
318
319 assert(src);
320 assert(src_size > 0);
321 assert(buffer);
322 assert(buffer_size);
323 assert(prefix);
324 assert(*buffer_size == 0 || *buffer);
325
326 if (src_size <= 8)
327 return -EBADMSG;
328
329 if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
330 return -ENOMEM;
331
332 r = LZ4_decompress_safe_partial((char*)src + 8, *buffer, src_size - 8,
333 prefix_len + 1, *buffer_size);
334 if (r >= 0)
335 size = (unsigned) r;
336 else {
337 /* lz4 always tries to decode full "sequence", so in
338 * pathological cases might need to decompress the
339 * full field. */
340 r = decompress_blob_lz4(src, src_size, buffer, buffer_size, &size, 0);
341 if (r < 0)
342 return r;
343 }
344
345 if (size >= prefix_len + 1)
346 return memcmp(*buffer, prefix, prefix_len) == 0 &&
347 ((const uint8_t*) *buffer)[prefix_len] == extra;
348 else
349 return 0;
350
351 #else
352 return -EPROTONOSUPPORT;
353 #endif
354 }
355
356 int decompress_startswith(int compression,
357 const void *src, uint64_t src_size,
358 void **buffer, size_t *buffer_size,
359 const void *prefix, size_t prefix_len,
360 uint8_t extra) {
361 if (compression == OBJECT_COMPRESSED_XZ)
362 return decompress_startswith_xz(src, src_size,
363 buffer, buffer_size,
364 prefix, prefix_len,
365 extra);
366 else if (compression == OBJECT_COMPRESSED_LZ4)
367 return decompress_startswith_lz4(src, src_size,
368 buffer, buffer_size,
369 prefix, prefix_len,
370 extra);
371 else
372 return -EBADMSG;
373 }
374
375 int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
376 #if HAVE_XZ
377 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
378 lzma_ret ret;
379 uint8_t buf[BUFSIZ], out[BUFSIZ];
380 lzma_action action = LZMA_RUN;
381
382 assert(fdf >= 0);
383 assert(fdt >= 0);
384
385 ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
386 if (ret != LZMA_OK) {
387 log_error("Failed to initialize XZ encoder: code %u", ret);
388 return -EINVAL;
389 }
390
391 for (;;) {
392 if (s.avail_in == 0 && action == LZMA_RUN) {
393 size_t m = sizeof(buf);
394 ssize_t n;
395
396 if (max_bytes != (uint64_t) -1 && (uint64_t) m > max_bytes)
397 m = (size_t) max_bytes;
398
399 n = read(fdf, buf, m);
400 if (n < 0)
401 return -errno;
402 if (n == 0)
403 action = LZMA_FINISH;
404 else {
405 s.next_in = buf;
406 s.avail_in = n;
407
408 if (max_bytes != (uint64_t) -1) {
409 assert(max_bytes >= (uint64_t) n);
410 max_bytes -= n;
411 }
412 }
413 }
414
415 if (s.avail_out == 0) {
416 s.next_out = out;
417 s.avail_out = sizeof(out);
418 }
419
420 ret = lzma_code(&s, action);
421 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END)) {
422 log_error("Compression failed: code %u", ret);
423 return -EBADMSG;
424 }
425
426 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
427 ssize_t n, k;
428
429 n = sizeof(out) - s.avail_out;
430
431 k = loop_write(fdt, out, n, false);
432 if (k < 0)
433 return k;
434
435 if (ret == LZMA_STREAM_END) {
436 log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
437 s.total_in, s.total_out,
438 (double) s.total_out / s.total_in * 100);
439
440 return 0;
441 }
442 }
443 }
444 #else
445 return -EPROTONOSUPPORT;
446 #endif
447 }
448
449 #define LZ4_BUFSIZE (512*1024u)
450
451 int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
452
453 #if HAVE_LZ4
454 LZ4F_errorCode_t c;
455 _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
456 _cleanup_free_ char *buf = NULL;
457 char *src = NULL;
458 size_t size, n, total_in = 0, total_out, offset = 0, frame_size;
459 struct stat st;
460 int r;
461 static const LZ4F_compressOptions_t options = {
462 .stableSrc = 1,
463 };
464 static const LZ4F_preferences_t preferences = {
465 .frameInfo.blockSizeID = 5,
466 };
467
468 c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
469 if (LZ4F_isError(c))
470 return -ENOMEM;
471
472 if (fstat(fdf, &st) < 0)
473 return log_debug_errno(errno, "fstat() failed: %m");
474
475 frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
476 size = frame_size + 64*1024; /* add some space for header and trailer */
477 buf = malloc(size);
478 if (!buf)
479 return -ENOMEM;
480
481 n = offset = total_out = LZ4F_compressBegin(ctx, buf, size, &preferences);
482 if (LZ4F_isError(n))
483 return -EINVAL;
484
485 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0);
486 if (src == MAP_FAILED)
487 return -errno;
488
489 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size, n);
490
491 while (total_in < (size_t) st.st_size) {
492 ssize_t k;
493
494 k = MIN(LZ4_BUFSIZE, st.st_size - total_in);
495 n = LZ4F_compressUpdate(ctx, buf + offset, size - offset,
496 src + total_in, k, &options);
497 if (LZ4F_isError(n)) {
498 r = -ENOTRECOVERABLE;
499 goto cleanup;
500 }
501
502 total_in += k;
503 offset += n;
504 total_out += n;
505
506 if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) {
507 log_debug("Compressed stream longer than %"PRIu64" bytes", max_bytes);
508 return -EFBIG;
509 }
510
511 if (size - offset < frame_size + 4) {
512 k = loop_write(fdt, buf, offset, false);
513 if (k < 0) {
514 r = k;
515 goto cleanup;
516 }
517 offset = 0;
518 }
519 }
520
521 n = LZ4F_compressEnd(ctx, buf + offset, size - offset, &options);
522 if (LZ4F_isError(n)) {
523 r = -ENOTRECOVERABLE;
524 goto cleanup;
525 }
526
527 offset += n;
528 total_out += n;
529 r = loop_write(fdt, buf, offset, false);
530 if (r < 0)
531 goto cleanup;
532
533 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
534 total_in, total_out,
535 (double) total_out / total_in * 100);
536 cleanup:
537 munmap(src, st.st_size);
538 return r;
539 #else
540 return -EPROTONOSUPPORT;
541 #endif
542 }
543
544 int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
545
546 #if HAVE_XZ
547 _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
548 lzma_ret ret;
549
550 uint8_t buf[BUFSIZ], out[BUFSIZ];
551 lzma_action action = LZMA_RUN;
552
553 assert(fdf >= 0);
554 assert(fdt >= 0);
555
556 ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
557 if (ret != LZMA_OK) {
558 log_debug("Failed to initialize XZ decoder: code %u", ret);
559 return -ENOMEM;
560 }
561
562 for (;;) {
563 if (s.avail_in == 0 && action == LZMA_RUN) {
564 ssize_t n;
565
566 n = read(fdf, buf, sizeof(buf));
567 if (n < 0)
568 return -errno;
569 if (n == 0)
570 action = LZMA_FINISH;
571 else {
572 s.next_in = buf;
573 s.avail_in = n;
574 }
575 }
576
577 if (s.avail_out == 0) {
578 s.next_out = out;
579 s.avail_out = sizeof(out);
580 }
581
582 ret = lzma_code(&s, action);
583 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END)) {
584 log_debug("Decompression failed: code %u", ret);
585 return -EBADMSG;
586 }
587
588 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
589 ssize_t n, k;
590
591 n = sizeof(out) - s.avail_out;
592
593 if (max_bytes != (uint64_t) -1) {
594 if (max_bytes < (uint64_t) n)
595 return -EFBIG;
596
597 max_bytes -= n;
598 }
599
600 k = loop_write(fdt, out, n, false);
601 if (k < 0)
602 return k;
603
604 if (ret == LZMA_STREAM_END) {
605 log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
606 s.total_in, s.total_out,
607 (double) s.total_out / s.total_in * 100);
608
609 return 0;
610 }
611 }
612 }
613 #else
614 log_debug("Cannot decompress file. Compiled without XZ support.");
615 return -EPROTONOSUPPORT;
616 #endif
617 }
618
619 int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
620 #if HAVE_LZ4
621 size_t c;
622 _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
623 _cleanup_free_ char *buf = NULL;
624 char *src;
625 struct stat st;
626 int r = 0;
627 size_t total_in = 0, total_out = 0;
628
629 c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
630 if (LZ4F_isError(c))
631 return -ENOMEM;
632
633 if (fstat(in, &st) < 0)
634 return log_debug_errno(errno, "fstat() failed: %m");
635
636 buf = malloc(LZ4_BUFSIZE);
637 if (!buf)
638 return -ENOMEM;
639
640 src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in, 0);
641 if (src == MAP_FAILED)
642 return -errno;
643
644 while (total_in < (size_t) st.st_size) {
645 size_t produced = LZ4_BUFSIZE;
646 size_t used = st.st_size - total_in;
647
648 c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
649 if (LZ4F_isError(c)) {
650 r = -EBADMSG;
651 goto cleanup;
652 }
653
654 total_in += used;
655 total_out += produced;
656
657 if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) {
658 log_debug("Decompressed stream longer than %"PRIu64" bytes", max_bytes);
659 r = -EFBIG;
660 goto cleanup;
661 }
662
663 r = loop_write(out, buf, produced, false);
664 if (r < 0)
665 goto cleanup;
666 }
667
668 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
669 total_in, total_out,
670 total_in > 0 ? (double) total_out / total_in * 100 : 0.0);
671 cleanup:
672 munmap(src, st.st_size);
673 return r;
674 #else
675 log_debug("Cannot decompress file. Compiled without LZ4 support.");
676 return -EPROTONOSUPPORT;
677 #endif
678 }
679
680 int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
681
682 if (endswith(filename, ".lz4"))
683 return decompress_stream_lz4(fdf, fdt, max_bytes);
684 else if (endswith(filename, ".xz"))
685 return decompress_stream_xz(fdf, fdt, max_bytes);
686 else
687 return -EPROTONOSUPPORT;
688 }