src/libpakfire/compress.c

   1 /*#############################################################################
   2 #                                                                             #
   3 # Pakfire - The IPFire package management system                              #
   4 # Copyright (C) 2021 Pakfire development team                                 #
   5 #                                                                             #
   6 # This program is free software: you can redistribute it and/or modify        #
   7 # it under the terms of the GNU General Public License as published by        #
   8 # the Free Software Foundation, either version 3 of the License, or           #
   9 # (at your option) any later version.                                         #
  10 #                                                                             #
  11 # This program is distributed in the hope that it will be useful,             #
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  14 # GNU General Public License for more details.                                #
  15 #                                                                             #
  16 # You should have received a copy of the GNU General Public License           #
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  18 #                                                                             #
  19 #############################################################################*/
  20
  21 #include <errno.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25
  26 #include <archive.h>
  27 #include <lzma.h>
  28 #include <zstd.h>
  29
  30 #include <pakfire/compress.h>
  31 #include <pakfire/file.h>
  32 #include <pakfire/filelist.h>
  33 #include <pakfire/logging.h>
  34 #include <pakfire/progressbar.h>
  35 #include <pakfire/util.h>
  36
  37 // Read up to N bytes for analyze the magic
  38 #define MAX_MAGIC_LENGTH        6
  39
  40 // Compression/Decompression buffer size
  41 #define BUFFER_SIZE                     64 * 1024
  42
  43 // Settings for XZ
  44 #define XZ_COMPRESSION_LEVEL    6
  45
  46 // Settings for ZSTD
  47 #define ZSTD_COMPRESSION_LEVEL  7
  48
  49 const struct compressor {
  50         char magic[MAX_MAGIC_LENGTH];
  51         size_t magic_length;
  52         FILE* (*open)(FILE* f, const char* mode);
  53 } compressors[] = {
  54         // XZ
  55         { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
  56         // ZSTD
  57         { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
  58         // End
  59         { "", 0, NULL, },
  60 };
  61
  62 // Try to guess the compression
  63 FILE* pakfire_xfopen(FILE* f, const char* mode) {
  64         char buffer[MAX_MAGIC_LENGTH];
  65
  66         if (!f) {
  67                 errno = EBADFD;
  68                 return NULL;
  69         }
  70
  71         if (!mode) {
  72                 errno = EINVAL;
  73                 return NULL;
  74         }
  75
  76         // This only works for reading files
  77         if (*mode != 'r') {
  78                 errno = ENOTSUP;
  79                 return NULL;
  80         }
  81
  82         fpos_t pos;
  83
  84         // Store the position
  85         int r = fgetpos(f, &pos);
  86         if (r < 0)
  87                 return NULL;
  88
  89         // Read a couple of bytes
  90         size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
  91
  92         // Reset position
  93         r = fsetpos(f, &pos);
  94         if (r < 0)
  95                 return NULL;
  96
  97         // Check if we could read anything
  98         if (!bytes_read || bytes_read < sizeof(buffer))
  99                 return f;
 100
 101         // Analyze magic
 102         for (const struct compressor* c = compressors; c->open; c++) {
 103                 // Check if we have read enough data
 104                 if (bytes_read < c->magic_length)
 105                         continue;
 106
 107                 // Compare the magic value
 108                 r = memcmp(c->magic, buffer, c->magic_length);
 109                 if (r)
 110                         continue;
 111
 112                 // We found a match!
 113                 return c->open(f, mode);
 114         }
 115
 116         // Nothing seems to match
 117         errno = ENOTSUP;
 118         return f;
 119 }
 120
 121 struct xz_cookie {
 122         FILE* f;
 123         char mode;
 124         lzma_stream stream;
 125         int done;
 126
 127         uint8_t buffer[BUFFER_SIZE];
 128 };
 129
 130 static ssize_t xz_read(void* data, char* buffer, size_t size) {
 131         struct xz_cookie* cookie = (struct xz_cookie*)data;
 132         if (!cookie)
 133                 return -1;
 134
 135         // Do not read when mode is "w"
 136         if (cookie->mode == 'w')
 137                 return -1;
 138
 139         lzma_action action = LZMA_RUN;
 140
 141         // Set output to allocated buffer
 142         cookie->stream.next_out  = (uint8_t *)buffer;
 143         cookie->stream.avail_out = size;
 144
 145         while (1) {
 146                 // Read something when the input buffer is empty
 147                 if (cookie->stream.avail_in == 0) {
 148                         cookie->stream.next_in  = cookie->buffer;
 149                         cookie->stream.avail_in = fread(cookie->buffer,
 150                                 1, sizeof(cookie->buffer), cookie->f);
 151
 152                         // Break if the input file could not be read
 153                         if (ferror(cookie->f))
 154                                 return -1;
 155
 156                         // Finish after we have reached the end of the input file
 157                         if (feof(cookie->f))
 158                                 cookie->done = 1;
 159                 }
 160
 161                 lzma_ret ret = lzma_code(&cookie->stream, action);
 162
 163                 // If the stream has ended, we just send the
 164                 // remaining output and mark that we are done.
 165                 if (ret == LZMA_STREAM_END) {
 166                         cookie->done = 1;
 167                         return size - cookie->stream.avail_out;
 168                 }
 169
 170                 // Break on all other unexpected errors
 171                 if (ret != LZMA_OK)
 172                         return -1;
 173
 174                 // When we have read enough to fill the entire output buffer, we return
 175                 if (cookie->stream.avail_out == 0)
 176                         return size;
 177
 178                 if (cookie->done)
 179                         return -1;
 180         }
 181 }
 182
 183 static ssize_t xz_write(void* data, const char* buffer, size_t size) {
 184         struct xz_cookie* cookie = (struct xz_cookie*)data;
 185         if (!cookie)
 186                 return -1;
 187
 188         // Do not write when mode is "r"
 189         if (cookie->mode == 'r')
 190                 return -1;
 191
 192         // Return nothing when there is no input
 193         if (size == 0)
 194                 return 0;
 195
 196         // Set input to allocated buffer
 197         cookie->stream.next_in  = (uint8_t *)buffer;
 198         cookie->stream.avail_in = size;
 199
 200         while (1) {
 201                 cookie->stream.next_out  = cookie->buffer;
 202                 cookie->stream.avail_out = sizeof(cookie->buffer);
 203
 204                 lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
 205                 if (ret != LZMA_OK)
 206                         return -1;
 207
 208                 size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 209                 if (bytes_to_write) {
 210                         size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 211
 212                         if (bytes_written != bytes_to_write)
 213                                 return -1;
 214                 }
 215
 216                 // Report that all data has been written
 217                 if (cookie->stream.avail_in == 0)
 218                         return size;
 219         }
 220 }
 221
 222 static int xz_close(void* data) {
 223         struct xz_cookie* cookie = (struct xz_cookie*)data;
 224         if (!cookie)
 225                 return -1;
 226
 227         if (cookie->mode == 'w') {
 228                 while (1) {
 229                         cookie->stream.next_out  = cookie->buffer;
 230                         cookie->stream.avail_out = sizeof(cookie->buffer);
 231
 232                         lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
 233                         if (ret != LZMA_OK && ret != LZMA_STREAM_END)
 234                                 return -1;
 235
 236                         size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 237                         if (bytes_to_write) {
 238                                 size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 239
 240                                 if (bytes_written != bytes_to_write)
 241                                         return -1;
 242                         }
 243
 244                         if (ret == LZMA_STREAM_END)
 245                                 break;
 246                 }
 247         }
 248
 249         lzma_end(&cookie->stream);
 250
 251         // Close input file
 252         int r = fclose(cookie->f);
 253         free(cookie);
 254
 255         return r;
 256 }
 257
 258 static cookie_io_functions_t xz_functions = {
 259         .read  = xz_read,
 260         .write = xz_write,
 261         .seek  = NULL,
 262         .close = xz_close,
 263 };
 264
 265 FILE* pakfire_xzfopen(FILE* f, const char* mode) {
 266         lzma_ret ret;
 267
 268         if (!f) {
 269                 errno = EBADFD;
 270                 return NULL;
 271         }
 272
 273         if (!mode) {
 274                 errno = EINVAL;
 275                 return NULL;
 276         }
 277
 278         struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
 279         if (!cookie)
 280                 return NULL;
 281
 282         cookie->f = f;
 283         cookie->mode = *mode;
 284
 285         switch (cookie->mode) {
 286                 case 'r':
 287                         ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
 288                         break;
 289
 290                 case 'w':
 291                         ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
 292                         break;
 293
 294                 default:
 295                         errno = ENOTSUP;
 296                         goto ERROR;
 297         }
 298
 299         if (ret != LZMA_OK)
 300                 goto ERROR;
 301
 302         return fopencookie(cookie, mode, xz_functions);
 303
 304 ERROR:
 305         free(cookie);
 306         return NULL;
 307 }
 308
 309 // ZSTD
 310
 311 struct zstd_cookie {
 312         FILE* f;
 313         char mode;
 314         int done;
 315
 316         // Encoder
 317         ZSTD_CStream* cstream;
 318         ZSTD_inBuffer in;
 319
 320         // Decoder
 321         ZSTD_DStream* dstream;
 322         ZSTD_outBuffer out;
 323
 324         uint8_t buffer[BUFFER_SIZE];
 325 };
 326
 327 static ssize_t zstd_read(void* data, char* buffer, size_t size) {
 328         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 329         if (!cookie)
 330                 return -1;
 331
 332         // Do not read when mode is "w"
 333         if (cookie->mode == 'w')
 334                 return -1;
 335
 336         if (cookie->done)
 337                 return 0;
 338
 339         size_t r = 0;
 340
 341         // Configure output buffer
 342         cookie->out.dst = buffer;
 343         cookie->out.pos = 0;
 344         cookie->out.size = size;
 345
 346         while (1) {
 347                 if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
 348                         cookie->in.pos = 0;
 349                         cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
 350                 }
 351
 352                 if (r || cookie->in.size)
 353                         r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
 354
 355                 if (r == 0 && feof(cookie->f)) {
 356                         cookie->done = 1;
 357                         return cookie->out.pos;
 358                 }
 359
 360                 if (ZSTD_isError(r))
 361                         return -1;
 362
 363                 // Buffer full
 364                 if (cookie->out.pos == size)
 365                         return size;
 366         }
 367 }
 368
 369 static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
 370         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 371         if (!cookie)
 372                 return -1;
 373
 374         // Do not write when mode is "r"
 375         if (cookie->mode == 'r')
 376                 return -1;
 377
 378         // Return nothing when there is no input
 379         if (size == 0)
 380                 return 0;
 381
 382         // Configure input buffer
 383         cookie->in.src = buffer;
 384         cookie->in.pos = 0;
 385         cookie->in.size = size;
 386
 387         while (1) {
 388                 cookie->out.pos = 0;
 389
 390                 size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
 391                 if (ZSTD_isError(r))
 392                         return -1;
 393
 394                 if (cookie->out.pos > 0) {
 395                         size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 396
 397                         if (bytes_written != cookie->out.pos)
 398                                 return -1;
 399                 }
 400
 401                 // Return when all input has been written
 402                 if (cookie->in.pos == size)
 403                         return size;
 404         }
 405 }
 406
 407 static int zstd_close(void* data) {
 408         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 409         if (!cookie)
 410                 return -1;
 411
 412         if (cookie->mode == 'w') {
 413                 while (1) {
 414                         // Reset output buffer
 415                         cookie->out.pos = 0;
 416
 417                         size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
 418                         if (ZSTD_isError(r))
 419                                 return -1;
 420
 421                         if (cookie->out.pos > 0) {
 422                                 size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 423
 424                                 if (bytes_written != cookie->out.pos)
 425                                         return -1;
 426                         }
 427
 428                         if (r == 0)
 429                                 break;
 430                 }
 431         }
 432
 433         int r = fclose(cookie->f);
 434
 435         // Free everything
 436         if (cookie->cstream)
 437                 ZSTD_freeCStream(cookie->cstream);
 438         if (cookie->dstream)
 439                 ZSTD_freeDStream(cookie->dstream);
 440         free(cookie);
 441
 442         return r;
 443 }
 444
 445 static cookie_io_functions_t zstd_functions = {
 446         .read  = zstd_read,
 447         .write = zstd_write,
 448         .seek  = NULL,
 449         .close = zstd_close,
 450 };
 451
 452 FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
 453         if (!f) {
 454                 errno = EBADFD;
 455                 return NULL;
 456         }
 457
 458         if (!mode) {
 459                 errno = EINVAL;
 460                 return NULL;
 461         }
 462
 463         struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
 464         if (!cookie)
 465                 return NULL;
 466
 467         cookie->f = f;
 468         cookie->mode = *mode;
 469
 470         size_t r;
 471         switch (cookie->mode) {
 472                 case 'r':
 473                         // Allocate stream
 474                         cookie->dstream = ZSTD_createDStream();
 475                         if (!cookie->dstream)
 476                                 goto ERROR;
 477
 478                         // Initialize stream
 479                         r = ZSTD_initDStream(cookie->dstream);
 480                         if (ZSTD_isError(r))
 481                                 goto ERROR;
 482
 483                         cookie->in.src = cookie->buffer;
 484                         cookie->in.pos = 0;
 485                         cookie->in.size = 0;
 486                         break;
 487
 488                 case 'w':
 489                         // Allocate stream
 490                         cookie->cstream = ZSTD_createCStream();
 491                         if (!cookie->cstream)
 492                                 goto ERROR;
 493
 494                         // Initialize stream
 495                         r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
 496                         if (ZSTD_isError(r))
 497                                 goto ERROR;
 498
 499                         cookie->out.dst = cookie->buffer;
 500                         cookie->out.pos = 0;
 501                         cookie->out.size = sizeof(cookie->buffer);
 502                         break;
 503
 504                 default:
 505                         errno = ENOTSUP;
 506                         goto ERROR;
 507         }
 508
 509         return fopencookie(cookie, mode, zstd_functions);
 510
 511 ERROR:
 512         if (cookie->cstream)
 513                 ZSTD_freeCStream(cookie->cstream);
 514         if (cookie->dstream)
 515                 ZSTD_freeDStream(cookie->dstream);
 516         free(cookie);
 517
 518         return NULL;
 519 }
 520
 521 // Common extraction
 522
 523 struct pakfire_extract {
 524         // Reference to Pakfire
 525         struct pakfire* pakfire;
 526
 527         // Flags
 528         int flags;
 529
 530         // The archive to extract
 531         struct archive* archive;
 532
 533         // The filelist of all extracted files
 534         struct pakfire_filelist* filelist;
 535
 536         // Prepend this prefix
 537         char prefix[PATH_MAX];
 538
 539         // The writer
 540         struct archive* writer;
 541
 542         // The progressbar
 543         struct pakfire_progressbar* progressbar;
 544 };
 545
 546 static int pakfire_extract_progressbar_create(struct pakfire_progressbar** progressbar,
 547                 const char* message, int flags) {
 548         int r;
 549
 550         // Create the progressbar
 551         r = pakfire_progressbar_create(progressbar, NULL);
 552         if (r)
 553                 return r;
 554
 555         // Add message
 556         if (message) {
 557                 r = pakfire_progressbar_add_string(*progressbar, "%s", message);
 558                 if (r)
 559                         return r;
 560         }
 561
 562         // Add bar
 563         r = pakfire_progressbar_add_bar(*progressbar);
 564         if (r)
 565                 return r;
 566
 567         // Add throughput
 568         if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT) {
 569                 r = pakfire_progressbar_add_transfer_speed(*progressbar);
 570                 if (r)
 571                         return r;
 572         }
 573
 574         // Add percentage
 575         r = pakfire_progressbar_add_percentage(*progressbar);
 576         if (r)
 577                 return r;
 578
 579         // Success
 580         return 0;
 581 }
 582
 583 static void pakfire_extract_progress(void* p) {
 584         struct pakfire_extract* data = (struct pakfire_extract*)p;
 585
 586         // Fetch how many bytes have been read
 587         const size_t position = archive_filter_bytes(data->archive, -1);
 588
 589         // Update progressbar
 590         pakfire_progressbar_update(data->progressbar, position);
 591 }
 592
 593 static int __pakfire_extract_entry(struct pakfire* pakfire, struct pakfire_extract* data,
 594                 struct archive_entry* entry) {
 595         struct pakfire_file* file = NULL;
 596         char buffer[PATH_MAX];
 597         int r;
 598
 599         // Fetch path
 600         const char* path = archive_entry_pathname(entry);
 601
 602         // Prepend the prefix
 603         if (*data->prefix) {
 604                 // Compose file path
 605                 r = pakfire_path_join(buffer, data->prefix, path);
 606                 if (r) {
 607                         ERROR(pakfire, "Could not compose file path: %m\n");
 608                         return r;
 609                 }
 610
 611                 // Set file path
 612                 archive_entry_set_pathname(entry, buffer);
 613
 614                 // Update hardlink destination
 615                 const char* link = archive_entry_hardlink(entry);
 616                 if (link) {
 617                         r = pakfire_path_join(buffer, data->prefix, link);
 618                         if (r) {
 619                                 ERROR(pakfire, "Could not compose hardlink path: %m\n");
 620                                 return r;
 621                         }
 622
 623                         // Set hardlink path
 624                         archive_entry_set_hardlink(entry, buffer);
 625                 }
 626         }
 627
 628         // Add entry to filelist (if requested)
 629         if (data->filelist) {
 630                 r = pakfire_file_create_from_archive_entry(&file, pakfire, entry);
 631                 if (r)
 632                         goto ERROR;
 633
 634                 // Append the file to the list
 635                 r = pakfire_filelist_append(data->filelist, file);
 636                 if (r)
 637                         goto ERROR;
 638         }
 639
 640         // Create file & extract payload
 641         if (data->writer) {
 642                 DEBUG(pakfire, "Extracting %s\n", path);
 643
 644                 r = archive_read_extract2(data->archive, entry, data->writer);
 645                 switch (r) {
 646                         case ARCHIVE_OK:
 647                                 r = 0;
 648                                 break;
 649
 650                         case ARCHIVE_WARN:
 651                                 ERROR(pakfire, "%s\n", archive_error_string(data->writer));
 652
 653                                 // Pretend everything has been okay
 654                                 r = 0;
 655                                 break;
 656
 657                         case ARCHIVE_FATAL:
 658                                 ERROR(pakfire, "%s\n", archive_error_string(data->writer));
 659                                 r = 1;
 660                                 break;
 661                 }
 662         }
 663
 664 ERROR:
 665         if (file)
 666                 pakfire_file_unref(file);
 667
 668         return r;
 669 }
 670
 671 int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
 672                 size_t size, struct pakfire_filelist* filelist,
 673                 const char* prefix, const char* message, int flags) {
 674         int r = 1;
 675
 676         struct pakfire_extract data = {
 677                 .pakfire  = pakfire,
 678                 .archive  = archive,
 679                 .filelist = filelist,
 680                 .flags    = flags,
 681                 .writer   = NULL,
 682         };
 683
 684         // Is this a dry run?
 685         const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;
 686
 687         // Should we show a progress bar?
 688         const int no_progress = flags & PAKFIRE_EXTRACT_NO_PROGRESS;
 689
 690         // Set prefix (including pakfire path)
 691         r = pakfire_path(pakfire, data.prefix, "%s", prefix);
 692         if (r)
 693                 goto ERROR;
 694
 695         // Allocate writer
 696         if (!dry_run) {
 697                 data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
 698                 if (!data.writer) {
 699                         ERROR(pakfire, "Could not create disk writer: %m\n");
 700                         r = 1;
 701                         goto ERROR;
 702                 }
 703         }
 704
 705         // Create the progressbar
 706         if (!no_progress) {
 707                 r = pakfire_extract_progressbar_create(&data.progressbar, message, flags);
 708                 if (r)
 709                         goto ERROR;
 710
 711                 // Register progress callback
 712                 archive_read_extract_set_progress_callback(data.archive,
 713                         pakfire_extract_progress, &data);
 714
 715                 // Start progressbar
 716                 pakfire_progressbar_start(data.progressbar, size);
 717         }
 718
 719         struct archive_entry* entry = NULL;
 720
 721         // Walk through the archive
 722         while (1) {
 723                 r = archive_read_next_header(archive, &entry);
 724
 725                 // End when we have reached the end of the archive
 726                 if (r == ARCHIVE_EOF) {
 727                         r = 0;
 728                         break;
 729                 }
 730
 731                 // Raise any other errors
 732                 else if (r) {
 733                         ERROR(pakfire, "Could not read next header: %s\n",
 734                                 archive_error_string(archive));
 735                         goto ERROR;
 736                 }
 737
 738                 // Extract the entry
 739                 r = __pakfire_extract_entry(pakfire, &data, entry);
 740                 if (r)
 741                         goto ERROR;
 742         }
 743
 744         // Finish the progressbar
 745         if (data.progressbar)
 746                 pakfire_progressbar_finish(data.progressbar);
 747
 748 ERROR:
 749         if (data.progressbar)
 750                 pakfire_progressbar_unref(data.progressbar);
 751         if (data.writer)
 752                 archive_write_free(data.writer);
 753
 754         return r;
 755 }