src/journal/compress.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <assert.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26
  27 #ifdef HAVE_XZ
  28 #  include <lzma.h>
  29 #endif
  30
  31 #ifdef HAVE_LZ4
  32 #  include <lz4.h>
  33 #endif
  34
  35 #include "compress.h"
  36 #include "macro.h"
  37 #include "util.h"
  38 #include "sparse-endian.h"
  39 #include "journal-def.h"
  40
  41 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
  42
  43 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
  44         [OBJECT_COMPRESSED_XZ] = "XZ",
  45         [OBJECT_COMPRESSED_LZ4] = "LZ4",
  46 };
  47
  48 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
  49
  50 int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
  51 #ifdef HAVE_XZ
  52         static const lzma_options_lzma opt = {
  53                 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
  54                 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4};
  55         static const lzma_filter filters[2] = {
  56                 {LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt},
  57                 {LZMA_VLI_UNKNOWN, NULL}
  58         };
  59         lzma_ret ret;
  60         size_t out_pos = 0;
  61
  62         assert(src);
  63         assert(src_size > 0);
  64         assert(dst);
  65         assert(dst_size);
  66
  67         /* Returns < 0 if we couldn't compress the data or the
  68          * compressed result is longer than the original */
  69
  70         if (src_size < 80)
  71                 return -ENOBUFS;
  72
  73         ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
  74                                         src, src_size, dst, &out_pos, src_size - 1);
  75         if (ret != LZMA_OK)
  76                 return -ENOBUFS;
  77
  78         *dst_size = out_pos;
  79         return 0;
  80 #else
  81         return -EPROTONOSUPPORT;
  82 #endif
  83 }
  84
  85 int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
  86 #ifdef HAVE_LZ4
  87         int r;
  88
  89         assert(src);
  90         assert(src_size > 0);
  91         assert(dst);
  92         assert(dst_size);
  93
  94         /* Returns < 0 if we couldn't compress the data or the
  95          * compressed result is longer than the original */
  96
  97         if (src_size < 9)
  98                 return -ENOBUFS;
  99
 100         r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
 101         if (r <= 0)
 102                 return -ENOBUFS;
 103
 104         *(le64_t*) dst = htole64(src_size);
 105         *dst_size = r + 8;
 106
 107         return 0;
 108 #else
 109         return -EPROTONOSUPPORT;
 110 #endif
 111 }
 112
 113
 114 int decompress_blob_xz(const void *src, uint64_t src_size,
 115                        void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
 116
 117 #ifdef HAVE_XZ
 118         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 119         lzma_ret ret;
 120         uint64_t space;
 121
 122         assert(src);
 123         assert(src_size > 0);
 124         assert(dst);
 125         assert(dst_alloc_size);
 126         assert(dst_size);
 127         assert(*dst_alloc_size == 0 || *dst);
 128
 129         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
 130         if (ret != LZMA_OK)
 131                 return -ENOMEM;
 132
 133         space = MIN(src_size * 2, dst_max ?: (uint64_t) -1);
 134         if (!greedy_realloc(dst, dst_alloc_size, space, 1))
 135                 return false;
 136
 137         s.next_in = src;
 138         s.avail_in = src_size;
 139
 140         s.next_out = *dst;
 141         s.avail_out = space;
 142
 143         for (;;) {
 144                 uint64_t used;
 145
 146                 ret = lzma_code(&s, LZMA_FINISH);
 147
 148                 if (ret == LZMA_STREAM_END)
 149                         break;
 150                 else if (ret != LZMA_OK)
 151                         return -ENOMEM;
 152
 153                 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
 154                         break;
 155                 else if (dst_max > 0 && space == dst_max)
 156                         return -ENOBUFS;
 157
 158                 used = space - s.avail_out;
 159                 space = MIN(2 * space, dst_max ?: (uint64_t) -1);
 160                 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
 161                         return false;
 162
 163                 s.avail_out = space - used;
 164                 s.next_out = *dst + used;
 165         }
 166
 167         *dst_size = space - s.avail_out;
 168         return 0;
 169 #else
 170         return -EPROTONOSUPPORT;
 171 #endif
 172 }
 173
 174 int decompress_blob_lz4(const void *src, uint64_t src_size,
 175                         void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
 176
 177 #ifdef HAVE_LZ4
 178         char* out;
 179         uint64_t size;
 180         int r;
 181
 182         assert(src);
 183         assert(src_size > 0);
 184         assert(dst);
 185         assert(dst_alloc_size);
 186         assert(dst_size);
 187         assert(*dst_alloc_size == 0 || *dst);
 188
 189         if (src_size <= 8)
 190                 return -EBADMSG;
 191
 192         size = le64toh( *(le64_t*)src );
 193         if (size > *dst_alloc_size) {
 194                 out = realloc(*dst, size);
 195                 if (!out)
 196                         return -ENOMEM;
 197                 *dst = out;
 198                 *dst_alloc_size = size;
 199         } else
 200                 out = *dst;
 201
 202         r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
 203         if (r < 0 || (uint64_t) r != size)
 204                 return -EBADMSG;
 205
 206         *dst_size = size;
 207         return 0;
 208 #else
 209         return -EPROTONOSUPPORT;
 210 #endif
 211 }
 212
 213 int decompress_blob(int compression,
 214                     const void *src, uint64_t src_size,
 215                     void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
 216         if (compression == OBJECT_COMPRESSED_XZ)
 217                 return decompress_blob_xz(src, src_size,
 218                                           dst, dst_alloc_size, dst_size, dst_max);
 219         else if (compression == OBJECT_COMPRESSED_LZ4)
 220                 return decompress_blob_lz4(src, src_size,
 221                                            dst, dst_alloc_size, dst_size, dst_max);
 222         else
 223                 return -EBADMSG;
 224 }
 225
 226
 227 int decompress_startswith_xz(const void *src, uint64_t src_size,
 228                              void **buffer, uint64_t *buffer_size,
 229                              const void *prefix, uint64_t prefix_len,
 230                              uint8_t extra) {
 231
 232 #ifdef HAVE_XZ
 233         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 234         lzma_ret ret;
 235
 236         /* Checks whether the decompressed blob starts with the
 237          * mentioned prefix. The byte extra needs to follow the
 238          * prefix */
 239
 240         assert(src);
 241         assert(src_size > 0);
 242         assert(buffer);
 243         assert(buffer_size);
 244         assert(prefix);
 245         assert(*buffer_size == 0 || *buffer);
 246
 247         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
 248         if (ret != LZMA_OK)
 249                 return -EBADMSG;
 250
 251         if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
 252                 return -ENOMEM;
 253
 254         s.next_in = src;
 255         s.avail_in = src_size;
 256
 257         s.next_out = *buffer;
 258         s.avail_out = *buffer_size;
 259
 260         for (;;) {
 261                 ret = lzma_code(&s, LZMA_FINISH);
 262
 263                 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
 264                         return -EBADMSG;
 265
 266                 if (*buffer_size - s.avail_out >= prefix_len + 1)
 267                         return memcmp(*buffer, prefix, prefix_len) == 0 &&
 268                                 ((const uint8_t*) *buffer)[prefix_len] == extra;
 269
 270                 if (ret == LZMA_STREAM_END)
 271                         return 0;
 272
 273                 s.avail_out += *buffer_size;
 274
 275                 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
 276                         return -ENOMEM;
 277
 278                 s.next_out = *buffer + *buffer_size - s.avail_out;
 279         }
 280
 281 #else
 282         return -EPROTONOSUPPORT;
 283 #endif
 284 }
 285
 286 int decompress_startswith_lz4(const void *src, uint64_t src_size,
 287                               void **buffer, uint64_t *buffer_size,
 288                               const void *prefix, uint64_t prefix_len,
 289                               uint8_t extra) {
 290 #ifdef HAVE_LZ4
 291         /* Checks whether the decompressed blob starts with the
 292          * mentioned prefix. The byte extra needs to follow the
 293          * prefix */
 294
 295         int r;
 296
 297         assert(src);
 298         assert(src_size > 0);
 299         assert(buffer);
 300         assert(buffer_size);
 301         assert(prefix);
 302         assert(*buffer_size == 0 || *buffer);
 303
 304         if (src_size <= 8)
 305                 return -EBADMSG;
 306
 307         if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
 308                 return -ENOMEM;
 309
 310         r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
 311                                         prefix_len + 1, *buffer_size);
 312
 313         if (r < 0)
 314                 return -EBADMSG;
 315         if ((unsigned) r >= prefix_len + 1)
 316                 return memcmp(*buffer, prefix, prefix_len) == 0 &&
 317                         ((const uint8_t*) *buffer)[prefix_len] == extra;
 318         else
 319                 return 0;
 320
 321 #else
 322         return -EPROTONOSUPPORT;
 323 #endif
 324 }
 325
 326 int decompress_startswith(int compression,
 327                           const void *src, uint64_t src_size,
 328                           void **buffer, uint64_t *buffer_size,
 329                           const void *prefix, uint64_t prefix_len,
 330                           uint8_t extra) {
 331         if (compression == OBJECT_COMPRESSED_XZ)
 332                 return decompress_startswith_xz(src, src_size,
 333                                                 buffer, buffer_size,
 334                                                 prefix, prefix_len,
 335                                                 extra);
 336         else if (compression == OBJECT_COMPRESSED_LZ4)
 337                 return decompress_startswith_lz4(src, src_size,
 338                                                  buffer, buffer_size,
 339                                                  prefix, prefix_len,
 340                                                  extra);
 341         else
 342                 return -EBADMSG;
 343 }
 344
 345 int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
 346 #ifdef HAVE_XZ
 347         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 348         lzma_ret ret;
 349
 350         uint8_t buf[BUFSIZ], out[BUFSIZ];
 351         lzma_action action = LZMA_RUN;
 352
 353         assert(fdf >= 0);
 354         assert(fdt >= 0);
 355
 356         ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
 357         if (ret != LZMA_OK) {
 358                 log_error("Failed to initialize XZ encoder: code %d", ret);
 359                 return -EINVAL;
 360         }
 361
 362         for (;;) {
 363                 if (s.avail_in == 0 && action == LZMA_RUN) {
 364                         size_t m = sizeof(buf);
 365                         ssize_t n;
 366
 367                         if (max_bytes != -1 && m > (size_t) max_bytes)
 368                                 m = max_bytes;
 369
 370                         n = read(fdf, buf, m);
 371                         if (n < 0)
 372                                 return -errno;
 373                         if (n == 0)
 374                                 action = LZMA_FINISH;
 375                         else {
 376                                 s.next_in = buf;
 377                                 s.avail_in = n;
 378
 379                                 if (max_bytes != -1) {
 380                                         assert(max_bytes >= n);
 381                                         max_bytes -= n;
 382                                 }
 383                         }
 384                 }
 385
 386                 if (s.avail_out == 0) {
 387                         s.next_out = out;
 388                         s.avail_out = sizeof(out);
 389                 }
 390
 391                 ret = lzma_code(&s, action);
 392                 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
 393                         log_error("Compression failed: code %d", ret);
 394                         return -EBADMSG;
 395                 }
 396
 397                 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
 398                         ssize_t n, k;
 399
 400                         n = sizeof(out) - s.avail_out;
 401
 402                         errno = 0;
 403                         k = loop_write(fdt, out, n, false);
 404                         if (k < 0)
 405                                 return k;
 406                         if (k != n)
 407                                 return errno ? -errno : -EIO;
 408
 409                         if (ret == LZMA_STREAM_END) {
 410                                 log_debug("XZ compression finished (%zu -> %zu bytes, %.1f%%)",
 411                                           s.total_in, s.total_out,
 412                                           (double) s.total_out / s.total_in * 100);
 413
 414                                 return 0;
 415                         }
 416                 }
 417         }
 418 #else
 419         return -EPROTONOSUPPORT;
 420 #endif
 421 }
 422
 423 #define LZ4_BUFSIZE (512*1024)
 424
 425 int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
 426
 427 #ifdef HAVE_LZ4
 428
 429         _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
 430         char *buf;
 431         LZ4_stream_t lz4_data = {};
 432         le32_t header;
 433         size_t total_in = 0, total_out = sizeof(header);
 434         ssize_t n;
 435
 436         assert(fdf >= 0);
 437         assert(fdt >= 0);
 438
 439         buf1 = malloc(LZ4_BUFSIZE);
 440         buf2 = malloc(LZ4_BUFSIZE);
 441         out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
 442         if (!buf1 || !buf2 || !out)
 443                 return log_oom();
 444
 445         buf = buf1;
 446         for (;;) {
 447                 size_t m;
 448                 int r;
 449
 450                 m = LZ4_BUFSIZE;
 451                 if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
 452                         m = max_bytes - total_in;
 453
 454                 n = read(fdf, buf, m);
 455                 if (n < 0)
 456                         return -errno;
 457                 if (n == 0)
 458                         break;
 459
 460                 total_in += n;
 461
 462                 r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
 463                 if (r == 0) {
 464                         log_debug("Compressed size exceeds original, aborting compression.");
 465                         return -ENOBUFS;
 466                 }
 467
 468                 header = htole32(r);
 469                 errno = 0;
 470
 471                 n = write(fdt, &header, sizeof(header));
 472                 if (n < 0)
 473                         return -errno;
 474                 if (n != sizeof(header))
 475                         return errno ? -errno : -EIO;
 476
 477                 n = loop_write(fdt, out, r, false);
 478                 if (n < 0)
 479                         return n;
 480                 if (n != r)
 481                         return errno ? -errno : -EIO;
 482
 483                 total_out += sizeof(header) + r;
 484
 485                 buf = buf == buf1 ? buf2 : buf1;
 486         }
 487
 488         header = htole32(0);
 489         n = write(fdt, &header, sizeof(header));
 490         if (n < 0)
 491                 return -errno;
 492         if (n != sizeof(header))
 493                 return errno ? -errno : -EIO;
 494
 495         log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
 496                   total_in, total_out,
 497                   (double) total_out / total_in * 100);
 498
 499         return 0;
 500 #else
 501         return -EPROTONOSUPPORT;
 502 #endif
 503 }
 504
 505 int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
 506
 507 #ifdef HAVE_XZ
 508         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 509         lzma_ret ret;
 510
 511         uint8_t buf[BUFSIZ], out[BUFSIZ];
 512         lzma_action action = LZMA_RUN;
 513
 514         assert(fdf >= 0);
 515         assert(fdt >= 0);
 516
 517         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
 518         if (ret != LZMA_OK) {
 519                 log_error("Failed to initialize XZ decoder: code %d", ret);
 520                 return -ENOMEM;
 521         }
 522
 523         for (;;) {
 524                 if (s.avail_in == 0 && action == LZMA_RUN) {
 525                         ssize_t n;
 526
 527                         n = read(fdf, buf, sizeof(buf));
 528                         if (n < 0)
 529                                 return -errno;
 530                         if (n == 0)
 531                                 action = LZMA_FINISH;
 532                         else {
 533                                 s.next_in = buf;
 534                                 s.avail_in = n;
 535                         }
 536                 }
 537
 538                 if (s.avail_out == 0) {
 539                         s.next_out = out;
 540                         s.avail_out = sizeof(out);
 541                 }
 542
 543                 ret = lzma_code(&s, action);
 544                 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
 545                         log_error("Decompression failed: code %d", ret);
 546                         return -EBADMSG;
 547                 }
 548
 549                 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
 550                         ssize_t n, k;
 551
 552                         n = sizeof(out) - s.avail_out;
 553
 554                         if (max_bytes != -1) {
 555                                 if (max_bytes < n)
 556                                         return -EFBIG;
 557
 558                                 max_bytes -= n;
 559                         }
 560
 561                         errno = 0;
 562                         k = loop_write(fdt, out, n, false);
 563                         if (k < 0)
 564                                 return k;
 565                         if (k != n)
 566                                 return errno ? -errno : -EIO;
 567
 568                         if (ret == LZMA_STREAM_END) {
 569                                 log_debug("XZ decompression finished (%zu -> %zu bytes, %.1f%%)",
 570                                           s.total_in, s.total_out,
 571                                           (double) s.total_out / s.total_in * 100);
 572
 573                                 return 0;
 574                         }
 575                 }
 576         }
 577 #else
 578         log_error("Cannot decompress file. Compiled without XZ support.");
 579         return -EPROTONOSUPPORT;
 580 #endif
 581 }
 582
 583 int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
 584
 585 #ifdef HAVE_LZ4
 586         _cleanup_free_ char *buf = NULL, *out = NULL;
 587         size_t buf_size = 0;
 588         LZ4_streamDecode_t lz4_data = {};
 589         le32_t header;
 590         size_t total_in = sizeof(header), total_out = 0;
 591
 592         assert(fdf >= 0);
 593         assert(fdt >= 0);
 594
 595         out = malloc(4*LZ4_BUFSIZE);
 596         if (!out)
 597                 return log_oom();
 598
 599         for (;;) {
 600                 ssize_t n, m;
 601                 int r;
 602
 603                 n = read(fdf, &header, sizeof(header));
 604                 if (n < 0)
 605                         return -errno;
 606                 if (n != sizeof(header))
 607                         return errno ? -errno : -EIO;
 608
 609                 m = le32toh(header);
 610                 if (m == 0)
 611                         break;
 612
 613                 /* We refuse to use a bigger decompression buffer than
 614                  * the one used for compression by 4 times. This means
 615                  * that compression buffer size can be enlarged 4
 616                  * times. This can be changed, but old binaries might
 617                  * not accept buffers compressed by newer binaries then.
 618                  */
 619                 if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
 620                         log_error("Compressed stream block too big: %zd bytes", m);
 621                         return -EBADMSG;
 622                 }
 623
 624                 total_in += sizeof(header) + m;
 625
 626                 if (!GREEDY_REALLOC(buf, buf_size, m))
 627                         return log_oom();
 628
 629                 errno = 0;
 630                 n = loop_read(fdf, buf, m, false);
 631                 if (n < 0)
 632                         return n;
 633                 if (n != m)
 634                         return errno ? -errno : -EIO;
 635
 636                 r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
 637                 if (r <= 0)
 638                         log_error("LZ4 decompression failed.");
 639
 640                 total_out += r;
 641
 642                 if (max_bytes != -1 && total_out > (size_t) max_bytes) {
 643                         log_debug("Decompressed stream longer than %zd bytes", max_bytes);
 644                         return -EFBIG;
 645                 }
 646
 647                 errno = 0;
 648                 n = loop_write(fdt, out, r, false);
 649                 if (n < 0)
 650                         return n;
 651                 if (n != r)
 652                         return errno ? -errno : -EIO;
 653         }
 654
 655         log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
 656                   total_in, total_out,
 657                   (double) total_out / total_in * 100);
 658
 659         return 0;
 660 #else
 661         log_error("Cannot decompress file. Compiled without LZ4 support.");
 662         return -EPROTONOSUPPORT;
 663 #endif
 664 }
 665
 666 int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
 667
 668         if (endswith(filename, ".lz4"))
 669                 return decompress_stream_lz4(fdf, fdt, max_bytes);
 670         else if (endswith(filename, ".xz"))
 671                 return decompress_stream_xz(fdf, fdt, max_bytes);
 672         else
 673                 return -EPROTONOSUPPORT;
 674 }