src/journal/compress.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <assert.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26
  27 #ifdef HAVE_XZ
  28 #  include <lzma.h>
  29 #endif
  30
  31 #ifdef HAVE_LZ4
  32 #  include <lz4.h>
  33 #endif
  34
  35 #include "compress.h"
  36 #include "macro.h"
  37 #include "util.h"
  38 #include "sparse-endian.h"
  39 #include "journal-def.h"
  40
  41 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
  42
  43 static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
  44         [OBJECT_COMPRESSED_XZ] = "XZ",
  45         [OBJECT_COMPRESSED_LZ4] = "LZ4",
  46 };
  47
  48 DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
  49
  50 int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
  51 #ifdef HAVE_XZ
  52         lzma_ret ret;
  53         size_t out_pos = 0;
  54
  55         assert(src);
  56         assert(src_size > 0);
  57         assert(dst);
  58         assert(dst_size);
  59
  60         /* Returns < 0 if we couldn't compress the data or the
  61          * compressed result is longer than the original */
  62
  63         ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_NONE, NULL,
  64                                       src, src_size, dst, &out_pos, src_size - 1);
  65         if (ret != LZMA_OK)
  66                 return -ENOBUFS;
  67
  68         *dst_size = out_pos;
  69         return 0;
  70 #else
  71         return -EPROTONOSUPPORT;
  72 #endif
  73 }
  74
  75 int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
  76 #ifdef HAVE_LZ4
  77         int r;
  78
  79         assert(src);
  80         assert(src_size > 0);
  81         assert(dst);
  82         assert(dst_size);
  83
  84         /* Returns < 0 if we couldn't compress the data or the
  85          * compressed result is longer than the original */
  86
  87         if (src_size < 9)
  88                 return -ENOBUFS;
  89
  90         r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
  91         if (r <= 0)
  92                 return -ENOBUFS;
  93
  94         *(le64_t*) dst = htole64(src_size);
  95         *dst_size = r + 8;
  96
  97         return 0;
  98 #else
  99         return -EPROTONOSUPPORT;
 100 #endif
 101 }
 102
 103
 104 int decompress_blob_xz(const void *src, uint64_t src_size,
 105                        void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
 106
 107 #ifdef HAVE_XZ
 108         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 109         lzma_ret ret;
 110         uint64_t space;
 111
 112         assert(src);
 113         assert(src_size > 0);
 114         assert(dst);
 115         assert(dst_alloc_size);
 116         assert(dst_size);
 117         assert(*dst_alloc_size == 0 || *dst);
 118
 119         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
 120         if (ret != LZMA_OK)
 121                 return -ENOMEM;
 122
 123         space = MIN(src_size * 2, dst_max ?: (uint64_t) -1);
 124         if (!greedy_realloc(dst, dst_alloc_size, space, 1))
 125                 return false;
 126
 127         s.next_in = src;
 128         s.avail_in = src_size;
 129
 130         s.next_out = *dst;
 131         s.avail_out = space;
 132
 133         for (;;) {
 134                 uint64_t used;
 135
 136                 ret = lzma_code(&s, LZMA_FINISH);
 137
 138                 if (ret == LZMA_STREAM_END)
 139                         break;
 140                 else if (ret != LZMA_OK)
 141                         return -ENOMEM;
 142
 143                 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
 144                         break;
 145                 else if (dst_max > 0 && space == dst_max)
 146                         return -ENOBUFS;
 147
 148                 used = space - s.avail_out;
 149                 space = MIN(2 * space, dst_max ?: (uint64_t) -1);
 150                 if (!greedy_realloc(dst, dst_alloc_size, space, 1))
 151                         return false;
 152
 153                 s.avail_out = space - used;
 154                 s.next_out = *dst + used;
 155         }
 156
 157         *dst_size = space - s.avail_out;
 158         return 0;
 159 #else
 160         return -EPROTONOSUPPORT;
 161 #endif
 162 }
 163
 164 int decompress_blob_lz4(const void *src, uint64_t src_size,
 165                         void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
 166
 167 #ifdef HAVE_LZ4
 168         char* out;
 169         uint64_t size;
 170         int r;
 171
 172         assert(src);
 173         assert(src_size > 0);
 174         assert(dst);
 175         assert(dst_alloc_size);
 176         assert(dst_size);
 177         assert(*dst_alloc_size == 0 || *dst);
 178
 179         if (src_size <= 8)
 180                 return -EBADMSG;
 181
 182         size = le64toh( *(le64_t*)src );
 183         if (size > *dst_alloc_size) {
 184                 out = realloc(*dst, size);
 185                 if (!out)
 186                         return -ENOMEM;
 187                 *dst = out;
 188                 *dst_alloc_size = size;
 189         } else
 190                 out = *dst;
 191
 192         r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
 193         if (r < 0 || (uint64_t) r != size)
 194                 return -EBADMSG;
 195
 196         *dst_size = size;
 197         return 0;
 198 #else
 199         return -EPROTONOSUPPORT;
 200 #endif
 201 }
 202
 203 int decompress_blob(int compression,
 204                     const void *src, uint64_t src_size,
 205                     void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
 206         if (compression == OBJECT_COMPRESSED_XZ)
 207                 return decompress_blob_xz(src, src_size,
 208                                           dst, dst_alloc_size, dst_size, dst_max);
 209         else if (compression == OBJECT_COMPRESSED_LZ4)
 210                 return decompress_blob_lz4(src, src_size,
 211                                            dst, dst_alloc_size, dst_size, dst_max);
 212         else
 213                 return -EBADMSG;
 214 }
 215
 216
 217 int decompress_startswith_xz(const void *src, uint64_t src_size,
 218                              void **buffer, uint64_t *buffer_size,
 219                              const void *prefix, uint64_t prefix_len,
 220                              uint8_t extra) {
 221
 222 #ifdef HAVE_XZ
 223         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 224         lzma_ret ret;
 225
 226         /* Checks whether the decompressed blob starts with the
 227          * mentioned prefix. The byte extra needs to follow the
 228          * prefix */
 229
 230         assert(src);
 231         assert(src_size > 0);
 232         assert(buffer);
 233         assert(buffer_size);
 234         assert(prefix);
 235         assert(*buffer_size == 0 || *buffer);
 236
 237         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
 238         if (ret != LZMA_OK)
 239                 return -EBADMSG;
 240
 241         if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
 242                 return -ENOMEM;
 243
 244         s.next_in = src;
 245         s.avail_in = src_size;
 246
 247         s.next_out = *buffer;
 248         s.avail_out = *buffer_size;
 249
 250         for (;;) {
 251                 ret = lzma_code(&s, LZMA_FINISH);
 252
 253                 if (ret != LZMA_STREAM_END && ret != LZMA_OK)
 254                         return -EBADMSG;
 255
 256                 if (*buffer_size - s.avail_out >= prefix_len + 1)
 257                         return memcmp(*buffer, prefix, prefix_len) == 0 &&
 258                                 ((const uint8_t*) *buffer)[prefix_len] == extra;
 259
 260                 if (ret == LZMA_STREAM_END)
 261                         return 0;
 262
 263                 s.avail_out += *buffer_size;
 264
 265                 if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
 266                         return -ENOMEM;
 267
 268                 s.next_out = *buffer + *buffer_size - s.avail_out;
 269         }
 270
 271 #else
 272         return -EPROTONOSUPPORT;
 273 #endif
 274 }
 275
 276 int decompress_startswith_lz4(const void *src, uint64_t src_size,
 277                               void **buffer, uint64_t *buffer_size,
 278                               const void *prefix, uint64_t prefix_len,
 279                               uint8_t extra) {
 280 #ifdef HAVE_LZ4
 281         /* Checks whether the decompressed blob starts with the
 282          * mentioned prefix. The byte extra needs to follow the
 283          * prefix */
 284
 285         int r;
 286
 287         assert(src);
 288         assert(src_size > 0);
 289         assert(buffer);
 290         assert(buffer_size);
 291         assert(prefix);
 292         assert(*buffer_size == 0 || *buffer);
 293
 294         if (src_size <= 8)
 295                 return -EBADMSG;
 296
 297         if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
 298                 return -ENOMEM;
 299
 300         r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
 301                                         prefix_len + 1, *buffer_size);
 302
 303         if (r < 0)
 304                 return -EBADMSG;
 305         if ((unsigned) r >= prefix_len + 1)
 306                 return memcmp(*buffer, prefix, prefix_len) == 0 &&
 307                         ((const uint8_t*) *buffer)[prefix_len] == extra;
 308         else
 309                 return 0;
 310
 311 #else
 312         return -EPROTONOSUPPORT;
 313 #endif
 314 }
 315
 316 int decompress_startswith(int compression,
 317                           const void *src, uint64_t src_size,
 318                           void **buffer, uint64_t *buffer_size,
 319                           const void *prefix, uint64_t prefix_len,
 320                           uint8_t extra) {
 321         if (compression == OBJECT_COMPRESSED_XZ)
 322                 return decompress_startswith_xz(src, src_size,
 323                                                 buffer, buffer_size,
 324                                                 prefix, prefix_len,
 325                                                 extra);
 326         else if (compression == OBJECT_COMPRESSED_LZ4)
 327                 return decompress_startswith_lz4(src, src_size,
 328                                                  buffer, buffer_size,
 329                                                  prefix, prefix_len,
 330                                                  extra);
 331         else
 332                 return -EBADMSG;
 333 }
 334
 335 int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
 336         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 337         lzma_ret ret;
 338
 339         uint8_t buf[BUFSIZ], out[BUFSIZ];
 340         lzma_action action = LZMA_RUN;
 341
 342         assert(fdf >= 0);
 343         assert(fdt >= 0);
 344
 345         ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
 346         if (ret != LZMA_OK) {
 347                 log_error("Failed to initialize XZ encoder: code %d", ret);
 348                 return -EINVAL;
 349         }
 350
 351         for (;;) {
 352                 if (s.avail_in == 0 && action == LZMA_RUN) {
 353                         size_t m = sizeof(buf);
 354                         ssize_t n;
 355
 356                         if (max_bytes != -1 && m > (size_t) max_bytes)
 357                                 m = max_bytes;
 358
 359                         n = read(fdf, buf, m);
 360                         if (n < 0)
 361                                 return -errno;
 362                         if (n == 0)
 363                                 action = LZMA_FINISH;
 364                         else {
 365                                 s.next_in = buf;
 366                                 s.avail_in = n;
 367
 368                                 if (max_bytes != -1) {
 369                                         assert(max_bytes >= n);
 370                                         max_bytes -= n;
 371                                 }
 372                         }
 373                 }
 374
 375                 if (s.avail_out == 0) {
 376                         s.next_out = out;
 377                         s.avail_out = sizeof(out);
 378                 }
 379
 380                 ret = lzma_code(&s, action);
 381                 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
 382                         log_error("Compression failed: code %d", ret);
 383                         return -EBADMSG;
 384                 }
 385
 386                 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
 387                         ssize_t n, k;
 388
 389                         n = sizeof(out) - s.avail_out;
 390
 391                         errno = 0;
 392                         k = loop_write(fdt, out, n, false);
 393                         if (k < 0)
 394                                 return k;
 395                         if (k != n)
 396                                 return errno ? -errno : -EIO;
 397
 398                         if (ret == LZMA_STREAM_END) {
 399                                 log_debug("XZ compression finished (%zu -> %zu bytes, %.1f%%)",
 400                                           s.total_in, s.total_out,
 401                                           (double) s.total_out / s.total_in * 100);
 402
 403                                 return 0;
 404                         }
 405                 }
 406         }
 407 }
 408
 409 #define LZ4_BUFSIZE (512*1024)
 410
 411 int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
 412
 413 #ifdef HAVE_LZ4
 414
 415         _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
 416         char *buf;
 417         LZ4_stream_t lz4_data = {};
 418         le32_t header;
 419         size_t total_in = 0, total_out = sizeof(header);
 420         ssize_t n;
 421
 422         assert(fdf >= 0);
 423         assert(fdt >= 0);
 424
 425         buf1 = malloc(LZ4_BUFSIZE);
 426         buf2 = malloc(LZ4_BUFSIZE);
 427         out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
 428         if (!buf1 || !buf2 || !out)
 429                 return log_oom();
 430
 431         buf = buf1;
 432         for (;;) {
 433                 size_t m;
 434                 int r;
 435
 436                 m = LZ4_BUFSIZE;
 437                 if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
 438                         m = max_bytes - total_in;
 439
 440                 n = read(fdf, buf, m);
 441                 if (n < 0)
 442                         return -errno;
 443                 if (n == 0)
 444                         break;
 445
 446                 total_in += n;
 447
 448                 r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
 449                 if (r == 0) {
 450                         log_debug("Compressed size exceeds original, aborting compression.");
 451                         return -ENOBUFS;
 452                 }
 453
 454                 header = htole32(r);
 455                 errno = 0;
 456
 457                 n = write(fdt, &header, sizeof(header));
 458                 if (n < 0)
 459                         return -errno;
 460                 if (n != sizeof(header))
 461                         return errno ? -errno : -EIO;
 462
 463                 n = loop_write(fdt, out, r, false);
 464                 if (n < 0)
 465                         return n;
 466                 if (n != r)
 467                         return errno ? -errno : -EIO;
 468
 469                 total_out += sizeof(header) + r;
 470
 471                 buf = buf == buf1 ? buf2 : buf1;
 472         }
 473
 474         header = htole32(0);
 475         n = write(fdt, &header, sizeof(header));
 476         if (n < 0)
 477                 return -errno;
 478         if (n != sizeof(header))
 479                 return errno ? -errno : -EIO;
 480
 481         log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
 482                   total_in, total_out,
 483                   (double) total_out / total_in * 100);
 484
 485         return 0;
 486 #else
 487         return -EPROTONOSUPPORT;
 488 #endif
 489 }
 490
 491 int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
 492
 493 #ifdef HAVE_XZ
 494         _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 495         lzma_ret ret;
 496
 497         uint8_t buf[BUFSIZ], out[BUFSIZ];
 498         lzma_action action = LZMA_RUN;
 499
 500         assert(fdf >= 0);
 501         assert(fdt >= 0);
 502
 503         ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
 504         if (ret != LZMA_OK) {
 505                 log_error("Failed to initialize XZ decoder: code %d", ret);
 506                 return -ENOMEM;
 507         }
 508
 509         for (;;) {
 510                 if (s.avail_in == 0 && action == LZMA_RUN) {
 511                         ssize_t n;
 512
 513                         n = read(fdf, buf, sizeof(buf));
 514                         if (n < 0)
 515                                 return -errno;
 516                         if (n == 0)
 517                                 action = LZMA_FINISH;
 518                         else {
 519                                 s.next_in = buf;
 520                                 s.avail_in = n;
 521                         }
 522                 }
 523
 524                 if (s.avail_out == 0) {
 525                         s.next_out = out;
 526                         s.avail_out = sizeof(out);
 527                 }
 528
 529                 ret = lzma_code(&s, action);
 530                 if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
 531                         log_error("Decompression failed: code %d", ret);
 532                         return -EBADMSG;
 533                 }
 534
 535                 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
 536                         ssize_t n, k;
 537
 538                         n = sizeof(out) - s.avail_out;
 539
 540                         if (max_bytes != -1) {
 541                                 if (max_bytes < n)
 542                                         return -EFBIG;
 543
 544                                 max_bytes -= n;
 545                         }
 546
 547                         errno = 0;
 548                         k = loop_write(fdt, out, n, false);
 549                         if (k < 0)
 550                                 return k;
 551                         if (k != n)
 552                                 return errno ? -errno : -EIO;
 553
 554                         if (ret == LZMA_STREAM_END) {
 555                                 log_debug("XZ decompression finished (%zu -> %zu bytes, %.1f%%)",
 556                                           s.total_in, s.total_out,
 557                                           (double) s.total_out / s.total_in * 100);
 558
 559                                 return 0;
 560                         }
 561                 }
 562         }
 563 #else
 564         log_error("Cannot decompress file. Compiled without XZ support.");
 565         return -EPROTONOSUPPORT;
 566 #endif
 567 }
 568
 569 int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
 570
 571 #ifdef HAVE_LZ4
 572         _cleanup_free_ char *buf = NULL, *out = NULL;
 573         size_t buf_size = 0;
 574         LZ4_streamDecode_t lz4_data = {};
 575         le32_t header;
 576         size_t total_in = sizeof(header), total_out = 0;
 577
 578         assert(fdf >= 0);
 579         assert(fdt >= 0);
 580
 581         out = malloc(4*LZ4_BUFSIZE);
 582         if (!out)
 583                 return log_oom();
 584
 585         for (;;) {
 586                 ssize_t n, m;
 587                 int r;
 588
 589                 n = read(fdf, &header, sizeof(header));
 590                 if (n < 0)
 591                         return -errno;
 592                 if (n != sizeof(header))
 593                         return errno ? -errno : -EIO;
 594
 595                 m = le32toh(header);
 596                 if (m == 0)
 597                         break;
 598
 599                 /* We refuse to use a bigger decompression buffer than
 600                  * the one used for compression by 4 times. This means
 601                  * that compression buffer size can be enlarged 4
 602                  * times. This can be changed, but old binaries might
 603                  * not accept buffers compressed by newer binaries then.
 604                  */
 605                 if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
 606                         log_error("Compressed stream block too big: %zd bytes", m);
 607                         return -EBADMSG;
 608                 }
 609
 610                 total_in += sizeof(header) + m;
 611
 612                 if (!GREEDY_REALLOC(buf, buf_size, m))
 613                         return log_oom();
 614
 615                 errno = 0;
 616                 n = loop_read(fdf, buf, m, false);
 617                 if (n < 0)
 618                         return n;
 619                 if (n != m)
 620                         return errno ? -errno : -EIO;
 621
 622                 r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
 623                 if (r <= 0)
 624                         log_error("LZ4 decompression failed.");
 625
 626                 total_out += r;
 627
 628                 if (max_bytes != -1 && total_out > (size_t) max_bytes) {
 629                         log_debug("Decompressed stream longer than %zd bytes", max_bytes);
 630                         return -EFBIG;
 631                 }
 632
 633                 errno = 0;
 634                 n = loop_write(fdt, out, r, false);
 635                 if (n < 0)
 636                         return n;
 637                 if (n != r)
 638                         return errno ? -errno : -EIO;
 639         }
 640
 641         log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
 642                   total_in, total_out,
 643                   (double) total_out / total_in * 100);
 644
 645         return 0;
 646 #else
 647         log_error("Cannot decompress file. Compiled without LZ4 support.");
 648         return -EPROTONOSUPPORT;
 649 #endif
 650 }
 651
 652 int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
 653
 654         if (endswith(filename, ".lz4"))
 655                 return decompress_stream_lz4(fdf, fdt, max_bytes);
 656         else if (endswith(filename, ".xz"))
 657                 return decompress_stream_xz(fdf, fdt, max_bytes);
 658         else
 659                 return -EPROTONOSUPPORT;
 660 }