src/journal/compress.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <stdlib.h>
  23 #include <string.h>
  24 #include <unistd.h>
  25
  26 #ifdef HAVE_XZ
  27 #  include <lzma.h>
  28 #endif
  29
  30 #ifdef HAVE_LZ4
  31 #  include <lz4.h>
  32 #endif
  33
  34 #include "compress.h"
  35 #include "macro.h"
  36 #include "util.h"
  37 #include "sparse-endian.h"
  38 #include "journal-def.h"
  39
  40 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
  41
  42 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
  43         [OBJECT_COMPRESSED_XZ] = "XZ",
  44         [OBJECT_COMPRESSED_LZ4] = "LZ4",
  45 };
  46
  47 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
  48
  49 int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
  50 #ifdef HAVE_XZ
  51         static const lzma_options_lzma opt = {
  52                 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
  53                 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4};
  54         static const lzma_filter filters[2] = {
  55                 {LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt},
  56                 {LZMA_VLI_UNKNOWN, NULL}
  57         };
  58         lzma_ret ret;
  59         size_t out_pos = 0;
  60
  61         assert(src);
  62         assert(src_size > 0);
  63         assert(dst);
  64         assert(dst_size);
  65
  66         /* Returns < 0 if we couldn't compress the data or the
  67          * compressed result is longer than the original */
  68
  69         if (src_size < 80)
  70                 return -ENOBUFS;
  71
  72         ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
  73                                         src, src_size, dst, &out_pos, src_size - 1);
  74         if (ret != LZMA_OK)
  75                 return -ENOBUFS;
  76
  77         *dst_size = out_pos;
  78         return 0;
  79 #else
  80         return -EPROTONOSUPPORT;
  81 #endif
  82 }
  83
  84 int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
  85 #ifdef HAVE_LZ4
  86         int r;
  87
  88         assert(src);
  89         assert(src_size > 0);
  90         assert(dst);
  91         assert(dst_size);
  92
  93         /* Returns < 0 if we couldn't compress the data or the
  94          * compressed result is longer than the original */
  95
  96         if (src_size < 9)
  97                 return -ENOBUFS;
  98
  99         r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
 100         if (r <= 0)
 101                 return -ENOBUFS;
 102
 103         *(le64_t*) dst = htole64(src_size);
 104         *dst_size = r + 8;
 105
 106         return 0;
 107 #else
 108         return -EPROTONOSUPPORT;
 109 #endif
 110 }
 111
 112
 113 int decompress_blob_xz(const void *src, uint64_t src_size,
 114                        void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
 115
 116 #ifdef HAVE_XZ
 117         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 118         lzma_ret ret;
 119         size_t space;
 120
 121         assert(src);
 122         assert(src_size > 0);
 123         assert(dst);
 124         assert(dst_alloc_size);
 125         assert(dst_size);
 126         assert(*dst_alloc_size == 0 || *dst);
 127
 128         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
 129         if (ret != LZMA_OK)
 130                 return -ENOMEM;
 131
 132         space = MIN(src_size * 2, dst_max ?: (size_t) -1);
 133         if (!greedy_realloc(dst, dst_alloc_size, space, 1))
 134                 return -ENOMEM;
 135
 136         s.next_in = src;
 137         s.avail_in = src_size;
 138
 139         s.next_out = *dst;
 140         s.avail_out = space;
 141
 142         for (;;) {
 143                 size_t used;
 144
 145                 ret = lzma_code(&s, LZMA_FINISH);
 146
 147                 if (ret == LZMA_STREAM_END)
 148                         break;
 149                 else if (ret != LZMA_OK)
 150                         return -ENOMEM;
 151
 152                 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
 153                         break;
 154                 else if (dst_max > 0 && space == dst_max)
 155                         return -ENOBUFS;
 156
 157                 used = space - s.avail_out;
 158                 space = MIN(2 * space, dst_max ?: (size_t) -1);
 159                 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
 160                         return -ENOMEM;
 161
 162                 s.avail_out = space - used;
 163                 s.next_out = *dst + used;
 164         }
 165
 166         *dst_size = space - s.avail_out;
 167         return 0;
 168 #else
 169         return -EPROTONOSUPPORT;
 170 #endif
 171 }
 172
 173 int decompress_blob_lz4(const void *src, uint64_t src_size,
 174                         void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
 175
 176 #ifdef HAVE_LZ4
 177         char* out;
 178         int r, size; /* LZ4 uses int for size */
 179
 180         assert(src);
 181         assert(src_size > 0);
 182         assert(dst);
 183         assert(dst_alloc_size);
 184         assert(dst_size);
 185         assert(*dst_alloc_size == 0 || *dst);
 186
 187         if (src_size <= 8)
 188                 return -EBADMSG;
 189
 190         size = le64toh( *(le64_t*)src );
 191         if (size < 0 || (le64_t) size != *(le64_t*)src)
 192                 return -EFBIG;
 193         if ((size_t) size > *dst_alloc_size) {
 194                 out = realloc(*dst, size);
 195                 if (!out)
 196                         return -ENOMEM;
 197                 *dst = out;
 198                 *dst_alloc_size = size;
 199         } else
 200                 out = *dst;
 201
 202         r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
 203         if (r < 0 || r != size)
 204                 return -EBADMSG;
 205
 206         *dst_size = size;
 207         return 0;
 208 #else
 209         return -EPROTONOSUPPORT;
 210 #endif
 211 }
 212
 213 int decompress_blob(int compression,
 214                     const void *src, uint64_t src_size,
 215                     void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
 216         if (compression == OBJECT_COMPRESSED_XZ)
 217                 return decompress_blob_xz(src, src_size,
 218                                           dst, dst_alloc_size, dst_size, dst_max);
 219         else if (compression == OBJECT_COMPRESSED_LZ4)
 220                 return decompress_blob_lz4(src, src_size,
 221                                            dst, dst_alloc_size, dst_size, dst_max);
 222         else
 223                 return -EBADMSG;
 224 }
 225
 226
 227 int decompress_startswith_xz(const void *src, uint64_t src_size,
 228                              void **buffer, size_t *buffer_size,
 229                              const void *prefix, size_t prefix_len,
 230                              uint8_t extra) {
 231
 232 #ifdef HAVE_XZ
 233         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 234         lzma_ret ret;
 235
 236         /* Checks whether the decompressed blob starts with the
 237          * mentioned prefix. The byte extra needs to follow the
 238          * prefix */
 239
 240         assert(src);
 241         assert(src_size > 0);
 242         assert(buffer);
 243         assert(buffer_size);
 244         assert(prefix);
 245         assert(*buffer_size == 0 || *buffer);
 246
 247         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
 248         if (ret != LZMA_OK)
 249                 return -EBADMSG;
 250
 251         if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
 252                 return -ENOMEM;
 253
 254         s.next_in = src;
 255         s.avail_in = src_size;
 256
 257         s.next_out = *buffer;
 258         s.avail_out = *buffer_size;
 259
 260         for (;;) {
 261                 ret = lzma_code(&s, LZMA_FINISH);
 262
 263                 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
 264                         return -EBADMSG;
 265
 266                 if (*buffer_size - s.avail_out >= prefix_len + 1)
 267                         return memcmp(*buffer, prefix, prefix_len) == 0 &&
 268                                 ((const uint8_t*) *buffer)[prefix_len] == extra;
 269
 270                 if (ret == LZMA_STREAM_END)
 271                         return 0;
 272
 273                 s.avail_out += *buffer_size;
 274
 275                 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
 276                         return -ENOMEM;
 277
 278                 s.next_out = *buffer + *buffer_size - s.avail_out;
 279         }
 280
 281 #else
 282         return -EPROTONOSUPPORT;
 283 #endif
 284 }
 285
 286 int decompress_startswith_lz4(const void *src, uint64_t src_size,
 287                               void **buffer, size_t *buffer_size,
 288                               const void *prefix, size_t prefix_len,
 289                               uint8_t extra) {
 290 #ifdef HAVE_LZ4
 291         /* Checks whether the decompressed blob starts with the
 292          * mentioned prefix. The byte extra needs to follow the
 293          * prefix */
 294
 295         int r;
 296
 297         assert(src);
 298         assert(src_size > 0);
 299         assert(buffer);
 300         assert(buffer_size);
 301         assert(prefix);
 302         assert(*buffer_size == 0 || *buffer);
 303
 304         if (src_size <= 8)
 305                 return -EBADMSG;
 306
 307         if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
 308                 return -ENOMEM;
 309
 310         r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
 311                                         prefix_len + 1, *buffer_size);
 312
 313         if (r < 0)
 314                 return -EBADMSG;
 315         if ((unsigned) r >= prefix_len + 1)
 316                 return memcmp(*buffer, prefix, prefix_len) == 0 &&
 317                         ((const uint8_t*) *buffer)[prefix_len] == extra;
 318         else
 319                 return 0;
 320
 321 #else
 322         return -EPROTONOSUPPORT;
 323 #endif
 324 }
 325
 326 int decompress_startswith(int compression,
 327                           const void *src, uint64_t src_size,
 328                           void **buffer, size_t *buffer_size,
 329                           const void *prefix, size_t prefix_len,
 330                           uint8_t extra) {
 331         if (compression == OBJECT_COMPRESSED_XZ)
 332                 return decompress_startswith_xz(src, src_size,
 333                                                 buffer, buffer_size,
 334                                                 prefix, prefix_len,
 335                                                 extra);
 336         else if (compression == OBJECT_COMPRESSED_LZ4)
 337                 return decompress_startswith_lz4(src, src_size,
 338                                                  buffer, buffer_size,
 339                                                  prefix, prefix_len,
 340                                                  extra);
 341         else
 342                 return -EBADMSG;
 343 }
 344
 345 int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
 346 #ifdef HAVE_XZ
 347         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 348         lzma_ret ret;
 349         uint8_t buf[BUFSIZ], out[BUFSIZ];
 350         lzma_action action = LZMA_RUN;
 351
 352         assert(fdf >= 0);
 353         assert(fdt >= 0);
 354
 355         ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
 356         if (ret != LZMA_OK) {
 357                 log_error("Failed to initialize XZ encoder: code %u", ret);
 358                 return -EINVAL;
 359         }
 360
 361         for (;;) {
 362                 if (s.avail_in == 0 && action == LZMA_RUN) {
 363                         size_t m = sizeof(buf);
 364                         ssize_t n;
 365
 366                         if (max_bytes != (uint64_t) -1 && (uint64_t) m > max_bytes)
 367                                 m = (size_t) max_bytes;
 368
 369                         n = read(fdf, buf, m);
 370                         if (n < 0)
 371                                 return -errno;
 372                         if (n == 0)
 373                                 action = LZMA_FINISH;
 374                         else {
 375                                 s.next_in = buf;
 376                                 s.avail_in = n;
 377
 378                                 if (max_bytes != (uint64_t) -1) {
 379                                         assert(max_bytes >= (uint64_t) n);
 380                                         max_bytes -= n;
 381                                 }
 382                         }
 383                 }
 384
 385                 if (s.avail_out == 0) {
 386                         s.next_out = out;
 387                         s.avail_out = sizeof(out);
 388                 }
 389
 390                 ret = lzma_code(&s, action);
 391                 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
 392                         log_error("Compression failed: code %u", ret);
 393                         return -EBADMSG;
 394                 }
 395
 396                 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
 397                         ssize_t n, k;
 398
 399                         n = sizeof(out) - s.avail_out;
 400
 401                         k = loop_write(fdt, out, n, false);
 402                         if (k < 0)
 403                                 return k;
 404
 405                         if (ret == LZMA_STREAM_END) {
 406                                 log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
 407                                           s.total_in, s.total_out,
 408                                           (double) s.total_out / s.total_in * 100);
 409
 410                                 return 0;
 411                         }
 412                 }
 413         }
 414 #else
 415         return -EPROTONOSUPPORT;
 416 #endif
 417 }
 418
 419 #define LZ4_BUFSIZE (512*1024)
 420
 421 int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
 422
 423 #ifdef HAVE_LZ4
 424
 425         _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
 426         char *buf;
 427         LZ4_stream_t lz4_data = {};
 428         le32_t header;
 429         size_t total_in = 0, total_out = sizeof(header);
 430         ssize_t n;
 431
 432         assert(fdf >= 0);
 433         assert(fdt >= 0);
 434
 435         buf1 = malloc(LZ4_BUFSIZE);
 436         buf2 = malloc(LZ4_BUFSIZE);
 437         out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
 438         if (!buf1 || !buf2 || !out)
 439                 return log_oom();
 440
 441         buf = buf1;
 442         for (;;) {
 443                 size_t m;
 444                 int r;
 445
 446                 m = LZ4_BUFSIZE;
 447                 if (max_bytes != (uint64_t) -1 && (uint64_t) m > (max_bytes - total_in))
 448                         m = (size_t) (max_bytes - total_in);
 449
 450                 n = read(fdf, buf, m);
 451                 if (n < 0)
 452                         return -errno;
 453                 if (n == 0)
 454                         break;
 455
 456                 total_in += n;
 457
 458                 r = LZ4_compress_continue(&lz4_data, buf, out, n);
 459                 if (r == 0) {
 460                         log_error("LZ4 compression failed.");
 461                         return -EBADMSG;
 462                 }
 463
 464                 header = htole32(r);
 465                 errno = 0;
 466
 467                 n = write(fdt, &header, sizeof(header));
 468                 if (n < 0)
 469                         return -errno;
 470                 if (n != sizeof(header))
 471                         return errno ? -errno : -EIO;
 472
 473                 n = loop_write(fdt, out, r, false);
 474                 if (n < 0)
 475                         return n;
 476
 477                 total_out += sizeof(header) + r;
 478
 479                 buf = buf == buf1 ? buf2 : buf1;
 480         }
 481
 482         header = htole32(0);
 483         n = write(fdt, &header, sizeof(header));
 484         if (n < 0)
 485                 return -errno;
 486         if (n != sizeof(header))
 487                 return errno ? -errno : -EIO;
 488
 489         log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
 490                   total_in, total_out,
 491                   (double) total_out / total_in * 100);
 492
 493         return 0;
 494 #else
 495         return -EPROTONOSUPPORT;
 496 #endif
 497 }
 498
 499 int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
 500
 501 #ifdef HAVE_XZ
 502         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 503         lzma_ret ret;
 504
 505         uint8_t buf[BUFSIZ], out[BUFSIZ];
 506         lzma_action action = LZMA_RUN;
 507
 508         assert(fdf >= 0);
 509         assert(fdt >= 0);
 510
 511         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
 512         if (ret != LZMA_OK) {
 513                 log_error("Failed to initialize XZ decoder: code %u", ret);
 514                 return -ENOMEM;
 515         }
 516
 517         for (;;) {
 518                 if (s.avail_in == 0 && action == LZMA_RUN) {
 519                         ssize_t n;
 520
 521                         n = read(fdf, buf, sizeof(buf));
 522                         if (n < 0)
 523                                 return -errno;
 524                         if (n == 0)
 525                                 action = LZMA_FINISH;
 526                         else {
 527                                 s.next_in = buf;
 528                                 s.avail_in = n;
 529                         }
 530                 }
 531
 532                 if (s.avail_out == 0) {
 533                         s.next_out = out;
 534                         s.avail_out = sizeof(out);
 535                 }
 536
 537                 ret = lzma_code(&s, action);
 538                 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
 539                         log_error("Decompression failed: code %u", ret);
 540                         return -EBADMSG;
 541                 }
 542
 543                 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
 544                         ssize_t n, k;
 545
 546                         n = sizeof(out) - s.avail_out;
 547
 548                         if (max_bytes != (uint64_t) -1) {
 549                                 if (max_bytes < (uint64_t) n)
 550                                         return -EFBIG;
 551
 552                                 max_bytes -= n;
 553                         }
 554
 555                         k = loop_write(fdt, out, n, false);
 556                         if (k < 0)
 557                                 return k;
 558
 559                         if (ret == LZMA_STREAM_END) {
 560                                 log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
 561                                           s.total_in, s.total_out,
 562                                           (double) s.total_out / s.total_in * 100);
 563
 564                                 return 0;
 565                         }
 566                 }
 567         }
 568 #else
 569         log_error("Cannot decompress file. Compiled without XZ support.");
 570         return -EPROTONOSUPPORT;
 571 #endif
 572 }
 573
 574 int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
 575
 576 #ifdef HAVE_LZ4
 577         _cleanup_free_ char *buf = NULL, *out = NULL;
 578         size_t buf_size = 0;
 579         LZ4_streamDecode_t lz4_data = {};
 580         le32_t header;
 581         size_t total_in = sizeof(header), total_out = 0;
 582
 583         assert(fdf >= 0);
 584         assert(fdt >= 0);
 585
 586         out = malloc(4*LZ4_BUFSIZE);
 587         if (!out)
 588                 return log_oom();
 589
 590         for (;;) {
 591                 ssize_t m;
 592                 int r;
 593
 594                 r = loop_read_exact(fdf, &header, sizeof(header), false);
 595                 if (r < 0)
 596                         return r;
 597
 598                 m = le32toh(header);
 599                 if (m == 0)
 600                         break;
 601
 602                 /* We refuse to use a bigger decompression buffer than
 603                  * the one used for compression by 4 times. This means
 604                  * that compression buffer size can be enlarged 4
 605                  * times. This can be changed, but old binaries might
 606                  * not accept buffers compressed by newer binaries then.
 607                  */
 608                 if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
 609                         log_error("Compressed stream block too big: %zd bytes", m);
 610                         return -EBADMSG;
 611                 }
 612
 613                 total_in += sizeof(header) + m;
 614
 615                 if (!GREEDY_REALLOC(buf, buf_size, m))
 616                         return log_oom();
 617
 618                 r = loop_read_exact(fdf, buf, m, false);
 619                 if (r < 0)
 620                         return r;
 621
 622                 r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
 623                 if (r <= 0)
 624                         log_error("LZ4 decompression failed.");
 625
 626                 total_out += r;
 627
 628                 if (max_bytes != (uint64_t) -1 && (uint64_t) total_out > max_bytes) {
 629                         log_debug("Decompressed stream longer than %" PRIu64 " bytes", max_bytes);
 630                         return -EFBIG;
 631                 }
 632
 633                 r = loop_write(fdt, out, r, false);
 634                 if (r < 0)
 635                         return r;
 636         }
 637
 638         log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
 639                   total_in, total_out,
 640                   (double) total_out / total_in * 100);
 641
 642         return 0;
 643 #else
 644         log_error("Cannot decompress file. Compiled without LZ4 support.");
 645         return -EPROTONOSUPPORT;
 646 #endif
 647 }
 648
 649 int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
 650
 651         if (endswith(filename, ".lz4"))
 652                 return decompress_stream_lz4(fdf, fdt, max_bytes);
 653         else if (endswith(filename, ".xz"))
 654                 return decompress_stream_xz(fdf, fdt, max_bytes);
 655         else
 656                 return -EPROTONOSUPPORT;
 657 }