src/libpakfire/compress.c

   1 /*#############################################################################
   2 #                                                                             #
   3 # Pakfire - The IPFire package management system                              #
   4 # Copyright (C) 2021 Pakfire development team                                 #
   5 #                                                                             #
   6 # This program is free software: you can redistribute it and/or modify        #
   7 # it under the terms of the GNU General Public License as published by        #
   8 # the Free Software Foundation, either version 3 of the License, or           #
   9 # (at your option) any later version.                                         #
  10 #                                                                             #
  11 # This program is distributed in the hope that it will be useful,             #
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  14 # GNU General Public License for more details.                                #
  15 #                                                                             #
  16 # You should have received a copy of the GNU General Public License           #
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  18 #                                                                             #
  19 #############################################################################*/
  20
  21 #include <errno.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25
  26 #include <archive.h>
  27 #include <lzma.h>
  28 #include <zstd.h>
  29
  30 #include <pakfire/compress.h>
  31 #include <pakfire/file.h>
  32 #include <pakfire/filelist.h>
  33 #include <pakfire/logging.h>
  34 #include <pakfire/progressbar.h>
  35 #include <pakfire/string.h>
  36 #include <pakfire/util.h>
  37
  38 // Read up to N bytes for analyze the magic
  39 #define MAX_MAGIC_LENGTH        6
  40
  41 // Compression/Decompression buffer size
  42 #define BUFFER_SIZE                     64 * 1024
  43
  44 // Settings for XZ
  45 #define XZ_COMPRESSION_LEVEL    6
  46
  47 // Settings for ZSTD
  48 #define ZSTD_COMPRESSION_LEVEL  7
  49
  50 const struct compressor {
  51         char magic[MAX_MAGIC_LENGTH];
  52         size_t magic_length;
  53         FILE* (*open)(FILE* f, const char* mode);
  54 } compressors[] = {
  55         // XZ
  56         { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
  57         // ZSTD
  58         { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
  59         // End
  60         { "", 0, NULL, },
  61 };
  62
  63 // Try to guess the compression
  64 FILE* pakfire_xfopen(FILE* f, const char* mode) {
  65         char buffer[MAX_MAGIC_LENGTH];
  66
  67         if (!f) {
  68                 errno = EBADFD;
  69                 return NULL;
  70         }
  71
  72         if (!mode) {
  73                 errno = EINVAL;
  74                 return NULL;
  75         }
  76
  77         // This only works for reading files
  78         if (*mode != 'r') {
  79                 errno = ENOTSUP;
  80                 return NULL;
  81         }
  82
  83         fpos_t pos;
  84
  85         // Store the position
  86         int r = fgetpos(f, &pos);
  87         if (r < 0)
  88                 return NULL;
  89
  90         // Read a couple of bytes
  91         size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
  92
  93         // Reset position
  94         r = fsetpos(f, &pos);
  95         if (r < 0)
  96                 return NULL;
  97
  98         // Check if we could read anything
  99         if (!bytes_read || bytes_read < sizeof(buffer))
 100                 return f;
 101
 102         // Analyze magic
 103         for (const struct compressor* c = compressors; c->open; c++) {
 104                 // Check if we have read enough data
 105                 if (bytes_read < c->magic_length)
 106                         continue;
 107
 108                 // Compare the magic value
 109                 r = memcmp(c->magic, buffer, c->magic_length);
 110                 if (r)
 111                         continue;
 112
 113                 // We found a match!
 114                 return c->open(f, mode);
 115         }
 116
 117         // Nothing seems to match
 118         errno = ENOTSUP;
 119         return f;
 120 }
 121
 122 struct xz_cookie {
 123         FILE* f;
 124         char mode;
 125         lzma_stream stream;
 126         int done;
 127
 128         uint8_t buffer[BUFFER_SIZE];
 129 };
 130
 131 static ssize_t xz_read(void* data, char* buffer, size_t size) {
 132         struct xz_cookie* cookie = (struct xz_cookie*)data;
 133         if (!cookie)
 134                 return -1;
 135
 136         // Do not read when mode is "w"
 137         if (cookie->mode == 'w')
 138                 return -1;
 139
 140         lzma_action action = LZMA_RUN;
 141
 142         // Set output to allocated buffer
 143         cookie->stream.next_out  = (uint8_t *)buffer;
 144         cookie->stream.avail_out = size;
 145
 146         while (1) {
 147                 // Read something when the input buffer is empty
 148                 if (cookie->stream.avail_in == 0) {
 149                         cookie->stream.next_in  = cookie->buffer;
 150                         cookie->stream.avail_in = fread(cookie->buffer,
 151                                 1, sizeof(cookie->buffer), cookie->f);
 152
 153                         // Break if the input file could not be read
 154                         if (ferror(cookie->f))
 155                                 return -1;
 156
 157                         // Finish after we have reached the end of the input file
 158                         if (feof(cookie->f))
 159                                 cookie->done = 1;
 160                 }
 161
 162                 lzma_ret ret = lzma_code(&cookie->stream, action);
 163
 164                 // If the stream has ended, we just send the
 165                 // remaining output and mark that we are done.
 166                 if (ret == LZMA_STREAM_END) {
 167                         cookie->done = 1;
 168                         return size - cookie->stream.avail_out;
 169                 }
 170
 171                 // Break on all other unexpected errors
 172                 if (ret != LZMA_OK)
 173                         return -1;
 174
 175                 // When we have read enough to fill the entire output buffer, we return
 176                 if (cookie->stream.avail_out == 0)
 177                         return size;
 178
 179                 if (cookie->done)
 180                         return -1;
 181         }
 182 }
 183
 184 static ssize_t xz_write(void* data, const char* buffer, size_t size) {
 185         struct xz_cookie* cookie = (struct xz_cookie*)data;
 186         if (!cookie)
 187                 return -1;
 188
 189         // Do not write when mode is "r"
 190         if (cookie->mode == 'r')
 191                 return -1;
 192
 193         // Return nothing when there is no input
 194         if (size == 0)
 195                 return 0;
 196
 197         // Set input to allocated buffer
 198         cookie->stream.next_in  = (uint8_t *)buffer;
 199         cookie->stream.avail_in = size;
 200
 201         while (1) {
 202                 cookie->stream.next_out  = cookie->buffer;
 203                 cookie->stream.avail_out = sizeof(cookie->buffer);
 204
 205                 lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
 206                 if (ret != LZMA_OK)
 207                         return -1;
 208
 209                 size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 210                 if (bytes_to_write) {
 211                         size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 212
 213                         if (bytes_written != bytes_to_write)
 214                                 return -1;
 215                 }
 216
 217                 // Report that all data has been written
 218                 if (cookie->stream.avail_in == 0)
 219                         return size;
 220         }
 221 }
 222
 223 static int xz_close(void* data) {
 224         struct xz_cookie* cookie = (struct xz_cookie*)data;
 225         if (!cookie)
 226                 return -1;
 227
 228         if (cookie->mode == 'w') {
 229                 while (1) {
 230                         cookie->stream.next_out  = cookie->buffer;
 231                         cookie->stream.avail_out = sizeof(cookie->buffer);
 232
 233                         lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
 234                         if (ret != LZMA_OK && ret != LZMA_STREAM_END)
 235                                 return -1;
 236
 237                         size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 238                         if (bytes_to_write) {
 239                                 size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 240
 241                                 if (bytes_written != bytes_to_write)
 242                                         return -1;
 243                         }
 244
 245                         if (ret == LZMA_STREAM_END)
 246                                 break;
 247                 }
 248         }
 249
 250         lzma_end(&cookie->stream);
 251
 252         // Close input file
 253         int r = fclose(cookie->f);
 254         free(cookie);
 255
 256         return r;
 257 }
 258
 259 static cookie_io_functions_t xz_functions = {
 260         .read  = xz_read,
 261         .write = xz_write,
 262         .seek  = NULL,
 263         .close = xz_close,
 264 };
 265
 266 FILE* pakfire_xzfopen(FILE* f, const char* mode) {
 267         lzma_ret ret;
 268
 269         if (!f) {
 270                 errno = EBADFD;
 271                 return NULL;
 272         }
 273
 274         if (!mode) {
 275                 errno = EINVAL;
 276                 return NULL;
 277         }
 278
 279         struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
 280         if (!cookie)
 281                 return NULL;
 282
 283         cookie->f = f;
 284         cookie->mode = *mode;
 285
 286         switch (cookie->mode) {
 287                 case 'r':
 288                         ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
 289                         break;
 290
 291                 case 'w':
 292                         ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
 293                         break;
 294
 295                 default:
 296                         errno = ENOTSUP;
 297                         goto ERROR;
 298         }
 299
 300         if (ret != LZMA_OK)
 301                 goto ERROR;
 302
 303         return fopencookie(cookie, mode, xz_functions);
 304
 305 ERROR:
 306         free(cookie);
 307         return NULL;
 308 }
 309
 310 // ZSTD
 311
 312 struct zstd_cookie {
 313         FILE* f;
 314         char mode;
 315         int done;
 316
 317         // Encoder
 318         ZSTD_CStream* cstream;
 319         ZSTD_inBuffer in;
 320
 321         // Decoder
 322         ZSTD_DStream* dstream;
 323         ZSTD_outBuffer out;
 324
 325         uint8_t buffer[BUFFER_SIZE];
 326 };
 327
 328 static ssize_t zstd_read(void* data, char* buffer, size_t size) {
 329         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 330         if (!cookie)
 331                 return -1;
 332
 333         // Do not read when mode is "w"
 334         if (cookie->mode == 'w')
 335                 return -1;
 336
 337         if (cookie->done)
 338                 return 0;
 339
 340         size_t r = 0;
 341
 342         // Configure output buffer
 343         cookie->out.dst = buffer;
 344         cookie->out.pos = 0;
 345         cookie->out.size = size;
 346
 347         while (1) {
 348                 if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
 349                         cookie->in.pos = 0;
 350                         cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
 351                 }
 352
 353                 if (r || cookie->in.size)
 354                         r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
 355
 356                 if (r == 0 && feof(cookie->f)) {
 357                         cookie->done = 1;
 358                         return cookie->out.pos;
 359                 }
 360
 361                 if (ZSTD_isError(r))
 362                         return -1;
 363
 364                 // Buffer full
 365                 if (cookie->out.pos == size)
 366                         return size;
 367         }
 368 }
 369
 370 static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
 371         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 372         if (!cookie)
 373                 return -1;
 374
 375         // Do not write when mode is "r"
 376         if (cookie->mode == 'r')
 377                 return -1;
 378
 379         // Return nothing when there is no input
 380         if (size == 0)
 381                 return 0;
 382
 383         // Configure input buffer
 384         cookie->in.src = buffer;
 385         cookie->in.pos = 0;
 386         cookie->in.size = size;
 387
 388         while (1) {
 389                 cookie->out.pos = 0;
 390
 391                 size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
 392                 if (ZSTD_isError(r))
 393                         return -1;
 394
 395                 if (cookie->out.pos > 0) {
 396                         size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 397
 398                         if (bytes_written != cookie->out.pos)
 399                                 return -1;
 400                 }
 401
 402                 // Return when all input has been written
 403                 if (cookie->in.pos == size)
 404                         return size;
 405         }
 406 }
 407
 408 static int zstd_close(void* data) {
 409         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 410         if (!cookie)
 411                 return -1;
 412
 413         if (cookie->mode == 'w') {
 414                 while (1) {
 415                         // Reset output buffer
 416                         cookie->out.pos = 0;
 417
 418                         size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
 419                         if (ZSTD_isError(r))
 420                                 return -1;
 421
 422                         if (cookie->out.pos > 0) {
 423                                 size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 424
 425                                 if (bytes_written != cookie->out.pos)
 426                                         return -1;
 427                         }
 428
 429                         if (r == 0)
 430                                 break;
 431                 }
 432         }
 433
 434         int r = fclose(cookie->f);
 435
 436         // Free everything
 437         if (cookie->cstream)
 438                 ZSTD_freeCStream(cookie->cstream);
 439         if (cookie->dstream)
 440                 ZSTD_freeDStream(cookie->dstream);
 441         free(cookie);
 442
 443         return r;
 444 }
 445
 446 static cookie_io_functions_t zstd_functions = {
 447         .read  = zstd_read,
 448         .write = zstd_write,
 449         .seek  = NULL,
 450         .close = zstd_close,
 451 };
 452
 453 FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
 454         if (!f) {
 455                 errno = EBADFD;
 456                 return NULL;
 457         }
 458
 459         if (!mode) {
 460                 errno = EINVAL;
 461                 return NULL;
 462         }
 463
 464         struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
 465         if (!cookie)
 466                 return NULL;
 467
 468         cookie->f = f;
 469         cookie->mode = *mode;
 470
 471         size_t r;
 472         switch (cookie->mode) {
 473                 case 'r':
 474                         // Allocate stream
 475                         cookie->dstream = ZSTD_createDStream();
 476                         if (!cookie->dstream)
 477                                 goto ERROR;
 478
 479                         // Initialize stream
 480                         r = ZSTD_initDStream(cookie->dstream);
 481                         if (ZSTD_isError(r))
 482                                 goto ERROR;
 483
 484                         cookie->in.src = cookie->buffer;
 485                         cookie->in.pos = 0;
 486                         cookie->in.size = 0;
 487                         break;
 488
 489                 case 'w':
 490                         // Allocate stream
 491                         cookie->cstream = ZSTD_createCStream();
 492                         if (!cookie->cstream)
 493                                 goto ERROR;
 494
 495                         // Initialize stream
 496                         r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
 497                         if (ZSTD_isError(r))
 498                                 goto ERROR;
 499
 500                         cookie->out.dst = cookie->buffer;
 501                         cookie->out.pos = 0;
 502                         cookie->out.size = sizeof(cookie->buffer);
 503                         break;
 504
 505                 default:
 506                         errno = ENOTSUP;
 507                         goto ERROR;
 508         }
 509
 510         return fopencookie(cookie, mode, zstd_functions);
 511
 512 ERROR:
 513         if (cookie->cstream)
 514                 ZSTD_freeCStream(cookie->cstream);
 515         if (cookie->dstream)
 516                 ZSTD_freeDStream(cookie->dstream);
 517         free(cookie);
 518
 519         return NULL;
 520 }
 521
 522 /*
 523         Helper function to conditionally walk through an archive
 524         and perform actions based on the callback.
 525 */
 526 int pakfire_walk(struct pakfire* pakfire, struct archive* archive,
 527                 pakfire_walk_callback callback, pakfire_walk_filter_callback filter_callback,
 528                 void* p) {
 529         struct archive_entry* entry = NULL;
 530         const char* path = NULL;
 531         int r;
 532
 533         // Walk through the archive
 534         for (;;) {
 535                 r = archive_read_next_header(archive, &entry);
 536
 537                 // Handle the return code
 538                 switch (r) {
 539                         // Fall through if everything is okay
 540                         case ARCHIVE_OK:
 541                                 break;
 542
 543                         // Return OK when we reached the end of the archive
 544                         case ARCHIVE_EOF:
 545                                 return 0;
 546
 547                         // Raise any other errors
 548                         default:
 549                                 return r;
 550                 }
 551
 552                 path = archive_entry_pathname(entry);
 553
 554                 DEBUG(pakfire, "Walking through %s...\n", path);
 555
 556                 // Call the filter callback before we call the actual callback
 557                 if (filter_callback) {
 558                         r = filter_callback(pakfire, archive, entry, p);
 559
 560                         // Handle the return code
 561                         switch (r) {
 562                                 case PAKFIRE_WALK_OK:
 563                                         break;
 564
 565                                 case PAKFIRE_WALK_END:
 566                                         DEBUG(pakfire, "Filter callback sent END\n");
 567                                         return 0;
 568
 569                                 case PAKFIRE_WALK_SKIP:
 570                                         DEBUG(pakfire, "Filter callback sent SKIP\n");
 571                                         continue;
 572
 573                                 case PAKFIRE_WALK_DONE:
 574                                         DEBUG(pakfire, "Filter callback sent DONE\n");
 575
 576                                         // Clear the callback function
 577                                         filter_callback = NULL;
 578                                         break;
 579
 580                                 // Raise any other errors
 581                                 default:
 582                                         DEBUG(pakfire, "Filter callback returned an error: %d\n", r);
 583                                         return r;
 584                         }
 585                 }
 586
 587                 // Run callback
 588                 if (callback) {
 589                         r = callback(pakfire, archive, entry, p);
 590
 591                         // Handle the return code
 592                         switch (r) {
 593                                 case PAKFIRE_WALK_OK:
 594                                         break;
 595
 596                                 case PAKFIRE_WALK_DONE:
 597                                         DEBUG(pakfire, "Callback sent DONE\n");
 598                                         return 0;
 599
 600                                 // Raise any other errors
 601                                 default:
 602                                         return r;
 603                         }
 604                 }
 605         }
 606
 607         return 0;
 608 }
 609
 610 // Common extraction
 611
 612 struct pakfire_extract {
 613         // Reference to Pakfire
 614         struct pakfire* pakfire;
 615
 616         // Flags
 617         int flags;
 618
 619         // The archive to extract
 620         struct archive* archive;
 621
 622         // The filelist of all extracted files
 623         struct pakfire_filelist* filelist;
 624
 625         // Prepend this prefix
 626         char prefix[PATH_MAX];
 627
 628         // The writer
 629         struct archive* writer;
 630
 631         // The progressbar
 632         struct pakfire_progressbar* progressbar;
 633 };
 634
 635 static int pakfire_extract_progressbar_create(struct pakfire_progressbar** progressbar,
 636                 const char* message, int flags) {
 637         int r;
 638
 639         // Create the progressbar
 640         r = pakfire_progressbar_create(progressbar, NULL);
 641         if (r)
 642                 return r;
 643
 644         // Add message
 645         if (message) {
 646                 r = pakfire_progressbar_add_string(*progressbar, "%s", message);
 647                 if (r)
 648                         return r;
 649         }
 650
 651         // Add bar
 652         r = pakfire_progressbar_add_bar(*progressbar);
 653         if (r)
 654                 return r;
 655
 656         // Add throughput
 657         if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT) {
 658                 r = pakfire_progressbar_add_transfer_speed(*progressbar);
 659                 if (r)
 660                         return r;
 661         }
 662
 663         // Add percentage
 664         r = pakfire_progressbar_add_percentage(*progressbar);
 665         if (r)
 666                 return r;
 667
 668         // Success
 669         return 0;
 670 }
 671
 672 static void pakfire_extract_progress(void* p) {
 673         struct pakfire_extract* data = (struct pakfire_extract*)p;
 674
 675         // Fetch how many bytes have been read
 676         const size_t position = archive_filter_bytes(data->archive, -1);
 677
 678         // Update progressbar
 679         pakfire_progressbar_update(data->progressbar, position);
 680 }
 681
 682 static int __pakfire_extract(struct pakfire* pakfire, struct archive* a,
 683                 struct archive_entry* entry, void* p) {
 684         struct pakfire_file* file = NULL;
 685         struct vfs_cap_data cap_data = {};
 686         char buffer[PATH_MAX];
 687         int r;
 688
 689         struct pakfire_extract* data = (struct pakfire_extract*)p;
 690
 691         // Fetch path
 692         const char* path = archive_entry_pathname(entry);
 693
 694         // Generate a file object
 695         r = pakfire_file_create_from_archive_entry(&file, pakfire, entry);
 696         if (r)
 697                 goto ERROR;
 698
 699         // Add entry to filelist (if requested)
 700         if (data->filelist) {
 701                 // Append the file to the list
 702                 r = pakfire_filelist_add(data->filelist, file);
 703                 if (r)
 704                         goto ERROR;
 705         }
 706
 707         const int configfile = pakfire_file_has_flag(file, PAKFIRE_FILE_CONFIG);
 708
 709         // Prepend the prefix
 710         if (*data->prefix) {
 711                 // Compose file path
 712                 r = pakfire_path_join(buffer, data->prefix, path);
 713                 if (r) {
 714                         ERROR(pakfire, "Could not compose file path: %m\n");
 715                         goto ERROR;
 716                 }
 717
 718                 // Set file path
 719                 archive_entry_set_pathname(entry, buffer);
 720
 721                 // Update hardlink destination
 722                 const char* link = archive_entry_hardlink(entry);
 723                 if (link) {
 724                         r = pakfire_path_join(buffer, data->prefix, link);
 725                         if (r) {
 726                                 ERROR(pakfire, "Could not compose hardlink path: %m\n");
 727                                 goto ERROR;
 728                         }
 729
 730                         // Set hardlink path
 731                         archive_entry_set_hardlink(entry, buffer);
 732                 }
 733         }
 734
 735         if (configfile) {
 736                 // Fetch path again since we changed it
 737                 path = archive_entry_pathname(entry);
 738
 739                 if (pakfire_path_exists(path)) {
 740                         DEBUG(pakfire, "The configuration file %s exists\n",
 741                                 pakfire_file_get_path(file));
 742
 743                         r = pakfire_string_format(buffer, "%s.paknew", path);
 744                         if (r) {
 745                                 ERROR(pakfire, "Could not compose path for configuration file: %m\n");
 746                                 goto ERROR;
 747                         }
 748
 749                         // Set the path again
 750                         archive_entry_set_pathname(entry, buffer);
 751                 }
 752         }
 753
 754         // Create file & extract payload
 755         if (data->writer) {
 756                 // Fetch path again since we changed it
 757                 path = archive_entry_pathname(entry);
 758
 759                 DEBUG(pakfire, "Extracting %s\n", path);
 760
 761                 // Remove any extended attributes which we never write to disk
 762                 archive_entry_xattr_clear(entry);
 763
 764                 // Set capabilities
 765                 if (pakfire_file_has_caps(file)) {
 766                         r = pakfire_file_write_fcaps(file, &cap_data);
 767                         if (r)
 768                                 goto ERROR;
 769
 770                         // Store capabilities in archive entry
 771                         archive_entry_xattr_add_entry(entry, "security.capability",
 772                                 &cap_data, sizeof(cap_data));
 773                 }
 774
 775                 // Write payload
 776                 r = archive_read_extract2(data->archive, entry, data->writer);
 777                 switch (r) {
 778                         case ARCHIVE_OK:
 779                                 r = 0;
 780                                 break;
 781
 782                         case ARCHIVE_WARN:
 783                                 ERROR(pakfire, "%s\n", archive_error_string(data->writer));
 784
 785                                 // Pretend everything has been okay
 786                                 r = 0;
 787                                 break;
 788
 789                         case ARCHIVE_FATAL:
 790                                 ERROR(pakfire, "%s\n", archive_error_string(data->writer));
 791                                 r = 1;
 792                                 break;
 793                 }
 794         }
 795
 796 ERROR:
 797         if (file)
 798                 pakfire_file_unref(file);
 799
 800         return r;
 801 }
 802
 803 int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
 804                 size_t size, struct pakfire_filelist* filelist,
 805                 const char* prefix, const char* message,
 806                 pakfire_walk_filter_callback filter_callback, int flags) {
 807         int r = 1;
 808
 809         // Use an empty string if no prefix set
 810         if (!prefix)
 811                 prefix = "";
 812
 813         struct pakfire_extract data = {
 814                 .pakfire  = pakfire,
 815                 .archive  = archive,
 816                 .filelist = filelist,
 817                 .flags    = flags,
 818                 .writer   = NULL,
 819         };
 820
 821         // Is this a dry run?
 822         const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;
 823
 824         // Should we show a progress bar?
 825         const int no_progress = flags & PAKFIRE_EXTRACT_NO_PROGRESS;
 826
 827         // Set prefix (including pakfire path)
 828         r = pakfire_path(pakfire, data.prefix, "%s", prefix);
 829         if (r)
 830                 goto ERROR;
 831
 832         // Allocate writer
 833         if (!dry_run) {
 834                 data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
 835                 if (!data.writer) {
 836                         ERROR(pakfire, "Could not create disk writer: %m\n");
 837                         r = 1;
 838                         goto ERROR;
 839                 }
 840         }
 841
 842         // Create the progressbar
 843         if (!no_progress) {
 844                 r = pakfire_extract_progressbar_create(&data.progressbar, message, flags);
 845                 if (r)
 846                         goto ERROR;
 847
 848                 // Register progress callback
 849                 archive_read_extract_set_progress_callback(data.archive,
 850                         pakfire_extract_progress, &data);
 851
 852                 // Start progressbar
 853                 pakfire_progressbar_start(data.progressbar, size);
 854         }
 855
 856         // Walk through the entire archive
 857         r = pakfire_walk(pakfire, archive, __pakfire_extract, filter_callback, &data);
 858         if (r)
 859                 goto ERROR;
 860
 861         // Finish the progressbar
 862         if (data.progressbar)
 863                 pakfire_progressbar_finish(data.progressbar);
 864
 865 ERROR:
 866         if (data.progressbar)
 867                 pakfire_progressbar_unref(data.progressbar);
 868         if (data.writer)
 869                 archive_write_free(data.writer);
 870
 871         return r;
 872 }
 873
 874 // Common compression
 875
 876 struct pakfire_compress {
 877         // Reference to Pakfire
 878         struct pakfire* pakfire;
 879
 880         // Flags
 881         int flags;
 882
 883         // The archive to write to
 884         struct archive* archive;
 885
 886         // Resolver for hardlinks
 887         struct archive_entry_linkresolver* linkresolver;
 888
 889         // The filelist of all files to write
 890         struct pakfire_filelist* filelist;
 891
 892         // The progressbar
 893         struct pakfire_progressbar* progressbar;
 894
 895         // Digests to write to the archive
 896         int digests;
 897 };
 898
 899 static int pakfire_compress_progressbar_create(struct pakfire_progressbar** progressbar,
 900                 const char* message, int flags) {
 901         int r;
 902
 903         // Create the progressbar
 904         r = pakfire_progressbar_create(progressbar, NULL);
 905         if (r)
 906                 return r;
 907
 908         // Add message
 909         if (message) {
 910                 r = pakfire_progressbar_add_string(*progressbar, "%s", message);
 911                 if (r)
 912                         return r;
 913         }
 914
 915         // Add bar
 916         r = pakfire_progressbar_add_bar(*progressbar);
 917         if (r)
 918                 return r;
 919
 920         // Add throughput
 921         if (flags & PAKFIRE_COMPRESS_SHOW_THROUGHPUT) {
 922                 r = pakfire_progressbar_add_transfer_speed(*progressbar);
 923                 if (r)
 924                         return r;
 925         }
 926
 927         // Show how many bytes have been written
 928         r = pakfire_progressbar_add_bytes_transferred(*progressbar);
 929         if (r)
 930                 return r;
 931
 932         // Add percentage
 933         r = pakfire_progressbar_add_percentage(*progressbar);
 934         if (r)
 935                 return r;
 936
 937         // Success
 938         return 0;
 939 }
 940
 941 static int __pakfire_compress_entry(struct pakfire* pakfire, struct pakfire_file* file,
 942                 struct pakfire_compress* data, struct archive_entry* entry) {
 943         FILE* f = NULL;
 944         int r;
 945
 946         // Write the header
 947         r = archive_write_header(data->archive, entry);
 948         if (r) {
 949                 ERROR(pakfire, "Error writing file header: %s\n",
 950                         archive_error_string(data->archive));
 951                 goto ERROR;
 952         }
 953
 954         // Copy the data if there is any
 955         if (archive_entry_size(entry)) {
 956                 // Open the file
 957                 f = pakfire_file_open(file);
 958                 if (!f) {
 959                         r = 1;
 960                         goto ERROR;
 961                 }
 962
 963                 // Copy the payload into the archive
 964                 r = pakfire_archive_copy_data_from_file(pakfire, data->archive, f);
 965                 if (r)
 966                         goto ERROR;
 967         }
 968
 969         // Write trailer
 970         r = archive_write_finish_entry(data->archive);
 971         if (r)
 972                 goto ERROR;
 973
 974 ERROR:
 975         if (f)
 976                 fclose(f);
 977
 978         return r;
 979 }
 980
 981 static int __pakfire_compress(struct pakfire* pakfire, struct pakfire_file* file, void* p) {
 982         struct archive_entry* entry = NULL;
 983         struct archive_entry* sparse_entry = NULL;
 984         int r = 1;
 985
 986         struct pakfire_compress* data = (struct pakfire_compress*)p;
 987
 988         // Fetch the file size
 989         const size_t size = pakfire_file_get_size(file);
 990
 991         // Generate file metadata into an archive entry
 992         entry = pakfire_file_archive_entry(file, data->digests);
 993         if (!entry) {
 994                 r = 1;
 995                 goto ERROR;
 996         }
 997
 998         // Perform search for hardlinks
 999         archive_entry_linkify(data->linkresolver, &entry, &sparse_entry);
1000
1001         // Write the main entry
1002         if (entry) {
1003                 r = __pakfire_compress_entry(pakfire, file, data, entry);
1004                 if (r)
1005                         goto ERROR;
1006         }
1007
1008         // Write the sparse entry
1009         if (sparse_entry) {
1010                 r = __pakfire_compress_entry(pakfire, file, data, sparse_entry);
1011                 if (r)
1012                         goto ERROR;
1013         }
1014
1015         // Query the link resolver for any more entries
1016         for (;;) {
1017                 // Free the entry
1018                 if (entry) {
1019                         archive_entry_free(entry);
1020                         entry = NULL;
1021                 }
1022
1023                 // Free the sparse entry
1024                 if (sparse_entry) {
1025                         archive_entry_free(sparse_entry);
1026                         sparse_entry = NULL;
1027                 }
1028
1029                 // Fetch the next entry
1030                 archive_entry_linkify(data->linkresolver, &entry, &sparse_entry);
1031                 if (!entry)
1032                         break;
1033
1034                 // Write the entry to the archive
1035                 r = __pakfire_compress_entry(pakfire, file, data, entry);
1036                 if (r)
1037                         goto ERROR;
1038         }
1039
1040         // Update the progressbar
1041         if (data->progressbar)
1042                 pakfire_progressbar_increment(data->progressbar, size);
1043
1044 ERROR:
1045         if (entry)
1046                 archive_entry_free(entry);
1047         if (sparse_entry)
1048                 archive_entry_free(sparse_entry);
1049
1050         return r;
1051 }
1052
1053 int pakfire_compress(struct pakfire* pakfire, struct archive* archive,
1054                 struct pakfire_filelist* filelist, const char* message, int flags, int digests) {
1055         int r = 1;
1056
1057         struct pakfire_compress data = {
1058                 .pakfire  = pakfire,
1059                 .archive  = archive,
1060                 .filelist = filelist,
1061                 .flags    = flags,
1062                 .digests  = digests,
1063         };
1064
1065         // Should we show a progress bar?
1066         const int no_progress = flags & PAKFIRE_COMPRESS_NO_PROGRESS;
1067
1068         // Fetch the length of the filelist
1069         const size_t size = pakfire_filelist_total_size(filelist);
1070
1071         // Create the progressbar
1072         if (!no_progress) {
1073                 r = pakfire_compress_progressbar_create(&data.progressbar, message, flags);
1074                 if (r)
1075                         goto ERROR;
1076
1077                 // Start progressbar
1078                 pakfire_progressbar_start(data.progressbar, size);
1079         }
1080
1081         // Setup the link resolver
1082         data.linkresolver = archive_entry_linkresolver_new();
1083         if (!data.linkresolver) {
1084                 ERROR(pakfire, "Could not setup link resolver: m\n");
1085                 goto ERROR;
1086         }
1087
1088         // Set the link resolver strategy
1089         archive_entry_linkresolver_set_strategy(data.linkresolver, archive_format(archive));
1090
1091         // Walk through the entire filelist
1092         r = pakfire_filelist_walk(filelist, __pakfire_compress, &data, 0);
1093         if (r)
1094                 goto ERROR;
1095
1096         // Finish the progressbar
1097         if (data.progressbar)
1098                 pakfire_progressbar_finish(data.progressbar);
1099
1100 ERROR:
1101         if (data.progressbar)
1102                 pakfire_progressbar_unref(data.progressbar);
1103         if (data.linkresolver)
1104                 archive_entry_linkresolver_free(data.linkresolver);
1105
1106         return r;
1107 }
1108
1109 int pakfire_compress_create_archive(struct pakfire* pakfire, struct archive** archive,
1110                 FILE* f, const enum pakfire_compressions compression, const unsigned int level) {
1111         struct archive* a = NULL;
1112         char value[16];
1113         int r;
1114
1115         // Open a new archive
1116         a = archive_write_new();
1117         if (!a) {
1118                 ERROR(pakfire, "archive_write_new() failed\n");
1119                 r = 1;
1120                 goto ERROR;
1121         }
1122
1123         // Use the PAX format
1124         r = archive_write_set_format_pax(a);
1125         if (r) {
1126                 ERROR(pakfire, "Could not set format to PAX: %s\n", archive_error_string(a));
1127                 goto ERROR;
1128         }
1129
1130         // Store any extended attributes in the SCHILY headers
1131         r = archive_write_set_format_option(a, "pax", "xattrheader", "SCHILY");
1132         if (r) {
1133                 ERROR(pakfire, "Could not set xattrheader option: %s\n", archive_error_string(a));
1134                 goto ERROR;
1135         }
1136
1137         switch (compression) {
1138                 case PAKFIRE_COMPRESS_ZSTD:
1139                         // Enable Zstd
1140                         r = archive_write_add_filter_zstd(a);
1141                         if (r) {
1142                                 ERROR(pakfire, "Could not enable Zstandard compression: %s\n",
1143                                         archive_error_string(a));
1144                                 goto ERROR;
1145                         }
1146
1147                         // Do not pad the last block
1148                         archive_write_set_bytes_in_last_block(a, 1);
1149
1150                         // Set compression level
1151                         if (level) {
1152                                 r = pakfire_string_format(value, "%u", level);
1153                                 if (r)
1154                                         goto ERROR;
1155
1156                                 r = archive_write_set_filter_option(a, NULL, "compression-level", value);
1157                                 if (r) {
1158                                         ERROR(pakfire, "Could not set Zstandard compression level: %s\n",
1159                                                 archive_error_string(a));
1160                                         goto ERROR;
1161                                 }
1162                         }
1163
1164 #if ARCHIVE_VERSION_NUMBER >= 3006000
1165                         // Fetch numbers of processors
1166                         long processors = sysconf(_SC_NPROCESSORS_ONLN);
1167
1168                         if (processors > 1) {
1169                                 r = pakfire_string_format(value, "%ld", processors);
1170                                 if (r) {
1171                                         ERROR(pakfire, "Could not format threads: %m\n");
1172                                         goto ERROR;
1173                                 }
1174
1175                                 // Try using multiple threads
1176                                 r = archive_write_set_filter_option(a, NULL, "threads", value);
1177                                 if (r) {
1178                                         ERROR(pakfire, "Could not enable %ld threads for compression: %s\n",
1179                                                 processors, archive_error_string(a));
1180                                         goto ERROR;
1181                                 }
1182                         }
1183 #endif
1184         }
1185
1186         // Write archive to f
1187         if (f) {
1188                 r = archive_write_open_FILE(a, f);
1189                 if (r) {
1190                         ERROR(pakfire, "archive_write_open_FILE() failed: %s\n", archive_error_string(a));
1191                         goto ERROR;
1192                 }
1193         }
1194
1195         // Success
1196         *archive = a;
1197
1198         return 0;
1199
1200 ERROR:
1201         if (a)
1202                 archive_write_free(a);
1203
1204         return r;
1205 }