src/libpakfire/compress.c

   1 /*#############################################################################
   2 #                                                                             #
   3 # Pakfire - The IPFire package management system                              #
   4 # Copyright (C) 2021 Pakfire development team                                 #
   5 #                                                                             #
   6 # This program is free software: you can redistribute it and/or modify        #
   7 # it under the terms of the GNU General Public License as published by        #
   8 # the Free Software Foundation, either version 3 of the License, or           #
   9 # (at your option) any later version.                                         #
  10 #                                                                             #
  11 # This program is distributed in the hope that it will be useful,             #
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  14 # GNU General Public License for more details.                                #
  15 #                                                                             #
  16 # You should have received a copy of the GNU General Public License           #
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  18 #                                                                             #
  19 #############################################################################*/
  20
  21 #include <errno.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25
  26 #include <archive.h>
  27 #include <lzma.h>
  28 #include <zstd.h>
  29
  30 #include <pakfire/ctx.h>
  31 #include <pakfire/compress.h>
  32 #include <pakfire/file.h>
  33 #include <pakfire/filelist.h>
  34 #include <pakfire/logging.h>
  35 #include <pakfire/path.h>
  36 #include <pakfire/progress.h>
  37 #include <pakfire/string.h>
  38 #include <pakfire/util.h>
  39
  40 // Read up to N bytes for analyze the magic
  41 #define MAX_MAGIC_LENGTH        6
  42
  43 // Compression/Decompression buffer size
  44 #define BUFFER_SIZE                     64 * 1024
  45
  46 // Settings for XZ
  47 #define XZ_COMPRESSION_LEVEL    6
  48
  49 // Settings for ZSTD
  50 #define ZSTD_COMPRESSION_LEVEL  7
  51
  52 const struct compressor {
  53         char magic[MAX_MAGIC_LENGTH];
  54         size_t magic_length;
  55         FILE* (*open)(FILE* f, const char* mode);
  56 } compressors[] = {
  57         // XZ
  58         { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
  59         // ZSTD
  60         { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
  61         // End
  62         { "", 0, NULL, },
  63 };
  64
  65 // Try to guess the compression
  66 FILE* pakfire_xfopen(FILE* f, const char* mode) {
  67         char buffer[MAX_MAGIC_LENGTH];
  68
  69         if (!f) {
  70                 errno = EBADFD;
  71                 return NULL;
  72         }
  73
  74         if (!mode) {
  75                 errno = EINVAL;
  76                 return NULL;
  77         }
  78
  79         // This only works for reading files
  80         if (*mode != 'r') {
  81                 errno = ENOTSUP;
  82                 return NULL;
  83         }
  84
  85         fpos_t pos;
  86
  87         // Store the position
  88         int r = fgetpos(f, &pos);
  89         if (r < 0)
  90                 return NULL;
  91
  92         // Read a couple of bytes
  93         size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
  94
  95         // Reset position
  96         r = fsetpos(f, &pos);
  97         if (r < 0)
  98                 return NULL;
  99
 100         // Check if we could read anything
 101         if (!bytes_read || bytes_read < sizeof(buffer))
 102                 return f;
 103
 104         // Analyze magic
 105         for (const struct compressor* c = compressors; c->open; c++) {
 106                 // Check if we have read enough data
 107                 if (bytes_read < c->magic_length)
 108                         continue;
 109
 110                 // Compare the magic value
 111                 r = memcmp(c->magic, buffer, c->magic_length);
 112                 if (r)
 113                         continue;
 114
 115                 // We found a match!
 116                 return c->open(f, mode);
 117         }
 118
 119         // Nothing seems to match
 120         errno = ENOTSUP;
 121         return f;
 122 }
 123
 124 struct xz_cookie {
 125         FILE* f;
 126         char mode;
 127         lzma_stream stream;
 128         int done;
 129
 130         uint8_t buffer[BUFFER_SIZE];
 131 };
 132
 133 static ssize_t xz_read(void* data, char* buffer, size_t size) {
 134         struct xz_cookie* cookie = (struct xz_cookie*)data;
 135         if (!cookie)
 136                 return -1;
 137
 138         // Do not read when mode is "w"
 139         if (cookie->mode == 'w')
 140                 return -1;
 141
 142         lzma_action action = LZMA_RUN;
 143
 144         // Set output to allocated buffer
 145         cookie->stream.next_out  = (uint8_t *)buffer;
 146         cookie->stream.avail_out = size;
 147
 148         while (1) {
 149                 // Read something when the input buffer is empty
 150                 if (cookie->stream.avail_in == 0) {
 151                         cookie->stream.next_in  = cookie->buffer;
 152                         cookie->stream.avail_in = fread(cookie->buffer,
 153                                 1, sizeof(cookie->buffer), cookie->f);
 154
 155                         // Break if the input file could not be read
 156                         if (ferror(cookie->f))
 157                                 return -1;
 158
 159                         // Finish after we have reached the end of the input file
 160                         if (feof(cookie->f))
 161                                 cookie->done = 1;
 162                 }
 163
 164                 lzma_ret ret = lzma_code(&cookie->stream, action);
 165
 166                 // If the stream has ended, we just send the
 167                 // remaining output and mark that we are done.
 168                 if (ret == LZMA_STREAM_END) {
 169                         cookie->done = 1;
 170                         return size - cookie->stream.avail_out;
 171                 }
 172
 173                 // Break on all other unexpected errors
 174                 if (ret != LZMA_OK)
 175                         return -1;
 176
 177                 // When we have read enough to fill the entire output buffer, we return
 178                 if (cookie->stream.avail_out == 0)
 179                         return size;
 180
 181                 if (cookie->done)
 182                         return -1;
 183         }
 184 }
 185
 186 static ssize_t xz_write(void* data, const char* buffer, size_t size) {
 187         struct xz_cookie* cookie = (struct xz_cookie*)data;
 188         if (!cookie)
 189                 return -1;
 190
 191         // Do not write when mode is "r"
 192         if (cookie->mode == 'r')
 193                 return -1;
 194
 195         // Return nothing when there is no input
 196         if (size == 0)
 197                 return 0;
 198
 199         // Set input to allocated buffer
 200         cookie->stream.next_in  = (uint8_t *)buffer;
 201         cookie->stream.avail_in = size;
 202
 203         while (1) {
 204                 cookie->stream.next_out  = cookie->buffer;
 205                 cookie->stream.avail_out = sizeof(cookie->buffer);
 206
 207                 lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
 208                 if (ret != LZMA_OK)
 209                         return -1;
 210
 211                 size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 212                 if (bytes_to_write) {
 213                         size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 214
 215                         if (bytes_written != bytes_to_write)
 216                                 return -1;
 217                 }
 218
 219                 // Report that all data has been written
 220                 if (cookie->stream.avail_in == 0)
 221                         return size;
 222         }
 223 }
 224
 225 static int xz_close(void* data) {
 226         struct xz_cookie* cookie = (struct xz_cookie*)data;
 227         if (!cookie)
 228                 return -1;
 229
 230         if (cookie->mode == 'w') {
 231                 while (1) {
 232                         cookie->stream.next_out  = cookie->buffer;
 233                         cookie->stream.avail_out = sizeof(cookie->buffer);
 234
 235                         lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
 236                         if (ret != LZMA_OK && ret != LZMA_STREAM_END)
 237                                 return -1;
 238
 239                         size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 240                         if (bytes_to_write) {
 241                                 size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 242
 243                                 if (bytes_written != bytes_to_write)
 244                                         return -1;
 245                         }
 246
 247                         if (ret == LZMA_STREAM_END)
 248                                 break;
 249                 }
 250         }
 251
 252         lzma_end(&cookie->stream);
 253
 254         // Close input file
 255         int r = fclose(cookie->f);
 256         free(cookie);
 257
 258         return r;
 259 }
 260
 261 static cookie_io_functions_t xz_functions = {
 262         .read  = xz_read,
 263         .write = xz_write,
 264         .seek  = NULL,
 265         .close = xz_close,
 266 };
 267
 268 FILE* pakfire_xzfopen(FILE* f, const char* mode) {
 269         lzma_ret ret;
 270
 271         if (!f) {
 272                 errno = EBADFD;
 273                 return NULL;
 274         }
 275
 276         if (!mode) {
 277                 errno = EINVAL;
 278                 return NULL;
 279         }
 280
 281         struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
 282         if (!cookie)
 283                 return NULL;
 284
 285         cookie->f = f;
 286         cookie->mode = *mode;
 287
 288         switch (cookie->mode) {
 289                 case 'r':
 290                         ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
 291                         break;
 292
 293                 case 'w':
 294                         ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
 295                         break;
 296
 297                 default:
 298                         errno = ENOTSUP;
 299                         goto ERROR;
 300         }
 301
 302         if (ret != LZMA_OK)
 303                 goto ERROR;
 304
 305         return fopencookie(cookie, mode, xz_functions);
 306
 307 ERROR:
 308         free(cookie);
 309         return NULL;
 310 }
 311
 312 // ZSTD
 313
 314 struct zstd_cookie {
 315         FILE* f;
 316         char mode;
 317         int done;
 318
 319         // Encoder
 320         ZSTD_CStream* cstream;
 321         ZSTD_inBuffer in;
 322
 323         // Decoder
 324         ZSTD_DStream* dstream;
 325         ZSTD_outBuffer out;
 326
 327         uint8_t buffer[BUFFER_SIZE];
 328 };
 329
 330 static ssize_t zstd_read(void* data, char* buffer, size_t size) {
 331         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 332         if (!cookie)
 333                 return -1;
 334
 335         // Do not read when mode is "w"
 336         if (cookie->mode == 'w')
 337                 return -1;
 338
 339         if (cookie->done)
 340                 return 0;
 341
 342         size_t r = 0;
 343
 344         // Configure output buffer
 345         cookie->out.dst = buffer;
 346         cookie->out.pos = 0;
 347         cookie->out.size = size;
 348
 349         while (1) {
 350                 if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
 351                         cookie->in.pos = 0;
 352                         cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
 353                 }
 354
 355                 if (r || cookie->in.size)
 356                         r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
 357
 358                 if (r == 0 && feof(cookie->f)) {
 359                         cookie->done = 1;
 360                         return cookie->out.pos;
 361                 }
 362
 363                 if (ZSTD_isError(r))
 364                         return -1;
 365
 366                 // Buffer full
 367                 if (cookie->out.pos == size)
 368                         return size;
 369         }
 370 }
 371
 372 static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
 373         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 374         if (!cookie)
 375                 return -1;
 376
 377         // Do not write when mode is "r"
 378         if (cookie->mode == 'r')
 379                 return -1;
 380
 381         // Return nothing when there is no input
 382         if (size == 0)
 383                 return 0;
 384
 385         // Configure input buffer
 386         cookie->in.src = buffer;
 387         cookie->in.pos = 0;
 388         cookie->in.size = size;
 389
 390         while (1) {
 391                 cookie->out.pos = 0;
 392
 393                 size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
 394                 if (ZSTD_isError(r))
 395                         return -1;
 396
 397                 if (cookie->out.pos > 0) {
 398                         size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 399
 400                         if (bytes_written != cookie->out.pos)
 401                                 return -1;
 402                 }
 403
 404                 // Return when all input has been written
 405                 if (cookie->in.pos == size)
 406                         return size;
 407         }
 408 }
 409
 410 static int zstd_close(void* data) {
 411         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 412         if (!cookie)
 413                 return -1;
 414
 415         if (cookie->mode == 'w') {
 416                 while (1) {
 417                         // Reset output buffer
 418                         cookie->out.pos = 0;
 419
 420                         size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
 421                         if (ZSTD_isError(r))
 422                                 return -1;
 423
 424                         if (cookie->out.pos > 0) {
 425                                 size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 426
 427                                 if (bytes_written != cookie->out.pos)
 428                                         return -1;
 429                         }
 430
 431                         if (r == 0)
 432                                 break;
 433                 }
 434         }
 435
 436         int r = fclose(cookie->f);
 437
 438         // Free everything
 439         if (cookie->cstream)
 440                 ZSTD_freeCStream(cookie->cstream);
 441         if (cookie->dstream)
 442                 ZSTD_freeDStream(cookie->dstream);
 443         free(cookie);
 444
 445         return r;
 446 }
 447
 448 static cookie_io_functions_t zstd_functions = {
 449         .read  = zstd_read,
 450         .write = zstd_write,
 451         .seek  = NULL,
 452         .close = zstd_close,
 453 };
 454
 455 FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
 456         if (!f) {
 457                 errno = EBADFD;
 458                 return NULL;
 459         }
 460
 461         if (!mode) {
 462                 errno = EINVAL;
 463                 return NULL;
 464         }
 465
 466         struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
 467         if (!cookie)
 468                 return NULL;
 469
 470         cookie->f = f;
 471         cookie->mode = *mode;
 472
 473         size_t r;
 474         switch (cookie->mode) {
 475                 case 'r':
 476                         // Allocate stream
 477                         cookie->dstream = ZSTD_createDStream();
 478                         if (!cookie->dstream)
 479                                 goto ERROR;
 480
 481                         // Initialize stream
 482                         r = ZSTD_initDStream(cookie->dstream);
 483                         if (ZSTD_isError(r))
 484                                 goto ERROR;
 485
 486                         cookie->in.src = cookie->buffer;
 487                         cookie->in.pos = 0;
 488                         cookie->in.size = 0;
 489                         break;
 490
 491                 case 'w':
 492                         // Allocate stream
 493                         cookie->cstream = ZSTD_createCStream();
 494                         if (!cookie->cstream)
 495                                 goto ERROR;
 496
 497                         // Initialize stream
 498                         r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
 499                         if (ZSTD_isError(r))
 500                                 goto ERROR;
 501
 502                         cookie->out.dst = cookie->buffer;
 503                         cookie->out.pos = 0;
 504                         cookie->out.size = sizeof(cookie->buffer);
 505                         break;
 506
 507                 default:
 508                         errno = ENOTSUP;
 509                         goto ERROR;
 510         }
 511
 512         return fopencookie(cookie, mode, zstd_functions);
 513
 514 ERROR:
 515         if (cookie->cstream)
 516                 ZSTD_freeCStream(cookie->cstream);
 517         if (cookie->dstream)
 518                 ZSTD_freeDStream(cookie->dstream);
 519         free(cookie);
 520
 521         return NULL;
 522 }
 523
 524 /*
 525         Helper function to conditionally walk through an archive
 526         and perform actions based on the callback.
 527 */
 528 int pakfire_walk(struct pakfire_ctx* ctx, struct archive* archive,
 529                 pakfire_walk_callback callback, pakfire_walk_filter_callback filter_callback,
 530                 void* p) {
 531         struct archive_entry* entry = NULL;
 532         int r;
 533
 534         // Walk through the archive
 535         for (;;) {
 536                 r = archive_read_next_header(archive, &entry);
 537
 538                 // Handle the return code
 539                 switch (r) {
 540                         // Fall through if everything is okay
 541                         case ARCHIVE_OK:
 542                                 break;
 543
 544                         // Return OK when we reached the end of the archive
 545                         case ARCHIVE_EOF:
 546                                 return 0;
 547
 548                         // Raise any other errors
 549                         default:
 550                                 return r;
 551                 }
 552
 553                 // Call the filter callback before we call the actual callback
 554                 if (filter_callback) {
 555                         r = filter_callback(ctx, archive, entry, p);
 556
 557                         // Handle the return code
 558                         switch (r) {
 559                                 case PAKFIRE_WALK_OK:
 560                                         break;
 561
 562                                 case PAKFIRE_WALK_END:
 563                                         CTX_DEBUG(ctx, "Filter callback sent END\n");
 564                                         return 0;
 565
 566                                 case PAKFIRE_WALK_SKIP:
 567                                         CTX_DEBUG(ctx, "Filter callback sent SKIP\n");
 568                                         continue;
 569
 570                                 case PAKFIRE_WALK_DONE:
 571                                         CTX_DEBUG(ctx, "Filter callback sent DONE\n");
 572
 573                                         // Clear the callback function
 574                                         filter_callback = NULL;
 575                                         break;
 576
 577                                 case PAKFIRE_WALK_AGAIN:
 578                                         CTX_DEBUG(ctx, "Filter callback sent AGAIN\n");
 579                                         return -EAGAIN;
 580
 581                                 // Raise any other errors
 582                                 default:
 583                                         CTX_DEBUG(ctx, "Filter callback returned an error: %d\n", r);
 584                                         return r;
 585                         }
 586                 }
 587
 588                 // Run callback
 589                 if (callback) {
 590                         r = callback(ctx, archive, entry, p);
 591
 592                         // Handle the return code
 593                         switch (r) {
 594                                 case PAKFIRE_WALK_OK:
 595                                         break;
 596
 597                                 case PAKFIRE_WALK_DONE:
 598                                         CTX_DEBUG(ctx, "Callback sent DONE\n");
 599                                         return 0;
 600
 601                                 // Raise any other errors
 602                                 default:
 603                                         return r;
 604                         }
 605                 }
 606         }
 607
 608         return 0;
 609 }
 610
 611 // Common extraction
 612
 613 struct pakfire_extract {
 614         // Reference to Pakfire
 615         struct pakfire* pakfire;
 616
 617         // Flags
 618         int flags;
 619
 620         // The archive to extract
 621         struct archive* archive;
 622
 623         // The filelist of all extracted files
 624         struct pakfire_filelist* filelist;
 625
 626         // Prepend this prefix
 627         const char* prefix;
 628
 629         // The writer
 630         struct archive* writer;
 631
 632         // The progress indicator
 633         struct pakfire_progress* progress;
 634 };
 635
 636 static void pakfire_extract_progress(void* p) {
 637         struct pakfire_extract* data = (struct pakfire_extract*)p;
 638
 639         // Fetch how many bytes have been read
 640         const size_t position = archive_filter_bytes(data->archive, -1);
 641
 642         // Update progress
 643         pakfire_progress_update(data->progress, position);
 644 }
 645
 646 static int __pakfire_extract(struct pakfire_ctx* ctx, struct archive* a,
 647                 struct archive_entry* entry, void* p) {
 648         struct pakfire_file* file = NULL;
 649         struct vfs_cap_data cap_data = {};
 650         char buffer[PATH_MAX];
 651         int r;
 652
 653         struct pakfire_extract* data = (struct pakfire_extract*)p;
 654
 655         // Fetch path
 656         const char* path = archive_entry_pathname(entry);
 657
 658         // Make sure we have a leading slash on the filelist
 659         if (!pakfire_string_startswith(path, "/")) {
 660                 r = pakfire_string_format(buffer, "/%s", path);
 661                 if (r)
 662                         goto ERROR;
 663
 664                 // Store the new name
 665                 archive_entry_set_pathname(entry, buffer);
 666
 667                 // Update the path pointer
 668                 path = archive_entry_pathname(entry);
 669         }
 670
 671         // Generate a file object
 672         r = pakfire_file_create_from_archive_entry(&file, data->pakfire, entry);
 673         if (r)
 674                 goto ERROR;
 675
 676         // Add entry to filelist (if requested)
 677         if (data->filelist) {
 678                 // Append the file to the list
 679                 r = pakfire_filelist_add(data->filelist, file);
 680                 if (r)
 681                         goto ERROR;
 682         }
 683
 684         const int configfile = pakfire_file_has_flag(file, PAKFIRE_FILE_CONFIG);
 685
 686         // Prepend the prefix
 687         if (*data->prefix) {
 688                 // Compose file path
 689                 r = pakfire_path_append(buffer, data->prefix, path);
 690                 if (r) {
 691                         CTX_ERROR(ctx, "Could not compose file path: %m\n");
 692                         goto ERROR;
 693                 }
 694
 695                 // Set file path
 696                 archive_entry_set_pathname(entry, buffer);
 697
 698                 // Update hardlink destination
 699                 const char* link = archive_entry_hardlink(entry);
 700                 if (link) {
 701                         r = pakfire_path_append(buffer, data->prefix, link);
 702                         if (r) {
 703                                 CTX_ERROR(ctx, "Could not compose hardlink path: %m\n");
 704                                 goto ERROR;
 705                         }
 706
 707                         // Set hardlink path
 708                         archive_entry_set_hardlink(entry, buffer);
 709                 }
 710         }
 711
 712         if (configfile) {
 713                 // Fetch path again since we changed it
 714                 path = archive_entry_pathname(entry);
 715
 716                 if (pakfire_path_exists(path)) {
 717                         CTX_DEBUG(ctx, "The configuration file %s exists\n",
 718                                 pakfire_file_get_path(file));
 719
 720                         r = pakfire_string_format(buffer, "%s.paknew", path);
 721                         if (r) {
 722                                 CTX_ERROR(ctx, "Could not compose path for configuration file: %m\n");
 723                                 goto ERROR;
 724                         }
 725
 726                         // Set the path again
 727                         archive_entry_set_pathname(entry, buffer);
 728                 }
 729         }
 730
 731         // Create file & extract payload
 732         if (data->writer) {
 733                 // Fetch path again since we changed it
 734                 path = archive_entry_pathname(entry);
 735
 736                 CTX_DEBUG(ctx, "Extracting %s\n", path);
 737
 738                 // Remove any extended attributes which we never write to disk
 739                 archive_entry_xattr_clear(entry);
 740
 741                 // Set capabilities
 742                 if (pakfire_file_has_caps(file)) {
 743                         r = pakfire_file_write_fcaps(file, &cap_data);
 744                         if (r)
 745                                 goto ERROR;
 746
 747                         // Store capabilities in archive entry
 748                         archive_entry_xattr_add_entry(entry, "security.capability",
 749                                 &cap_data, sizeof(cap_data));
 750                 }
 751
 752                 // Write payload
 753                 r = archive_read_extract2(data->archive, entry, data->writer);
 754                 switch (r) {
 755                         case ARCHIVE_OK:
 756                                 r = 0;
 757                                 break;
 758
 759                         case ARCHIVE_WARN:
 760                                 CTX_ERROR(ctx, "%s\n", archive_error_string(data->writer));
 761
 762                                 // Pretend everything has been okay
 763                                 r = 0;
 764                                 break;
 765
 766                         case ARCHIVE_FATAL:
 767                                 CTX_ERROR(ctx, "%s\n", archive_error_string(data->writer));
 768                                 r = 1;
 769                                 break;
 770                 }
 771         }
 772
 773 ERROR:
 774         if (file)
 775                 pakfire_file_unref(file);
 776
 777         return r;
 778 }
 779
 780 int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
 781                 size_t size, struct pakfire_filelist* filelist,
 782                 const char* prefix, const char* message,
 783                 pakfire_walk_filter_callback filter_callback, int flags) {
 784         int progress_flags = PAKFIRE_PROGRESS_SHOW_PERCENTAGE;
 785         int r = 1;
 786
 787         struct pakfire_ctx* ctx = pakfire_ctx(pakfire);
 788
 789         // Use / if no prefix is set
 790         if (!prefix)
 791                 prefix = "/";
 792
 793         struct pakfire_extract data = {
 794                 .pakfire  = pakfire,
 795                 .archive  = archive,
 796                 .filelist = filelist,
 797                 .prefix   = prefix,
 798                 .flags    = flags,
 799                 .writer   = NULL,
 800         };
 801
 802         // Is this a dry run?
 803         const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;
 804
 805         // Allocate writer
 806         if (!dry_run) {
 807                 data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
 808                 if (!data.writer) {
 809                         CTX_ERROR(ctx, "Could not create disk writer: %m\n");
 810                         r = 1;
 811                         goto ERROR;
 812                 }
 813         }
 814
 815         // Should we show any progress?
 816         if (flags & PAKFIRE_EXTRACT_NO_PROGRESS)
 817                 progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
 818
 819         // Show throughput?
 820         if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT)
 821                 progress_flags |= PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED;
 822
 823         // Create the progress indicator
 824         r = pakfire_progress_create(&data.progress, ctx, progress_flags, NULL);
 825         if (r)
 826                 goto ERROR;
 827
 828         // Set the title
 829         r = pakfire_progress_set_title(data.progress, "%s", message);
 830         if (r)
 831                 goto ERROR;
 832
 833         // Register progress callback
 834         archive_read_extract_set_progress_callback(data.archive,
 835                 pakfire_extract_progress, &data);
 836
 837         // Start progress
 838         r = pakfire_progress_start(data.progress, size);
 839         if (r)
 840                 goto ERROR;
 841
 842         // Walk through the entire archive
 843         r = pakfire_walk(ctx, archive, __pakfire_extract, filter_callback, &data);
 844         if (r)
 845                 goto ERROR;
 846
 847         // Finish the progress
 848         r = pakfire_progress_finish(data.progress);
 849         if (r)
 850                 goto ERROR;
 851
 852 ERROR:
 853         if (data.progress)
 854                 pakfire_progress_unref(data.progress);
 855         if (data.writer)
 856                 archive_write_free(data.writer);
 857         if (ctx)
 858                 pakfire_ctx_unref(ctx);
 859
 860         return r;
 861 }
 862
 863 // Common compression
 864
 865 struct pakfire_compress {
 866         // Reference to context
 867         struct pakfire_ctx* ctx;
 868
 869         // Reference to Pakfire
 870         struct pakfire* pakfire;
 871
 872         // Flags
 873         int flags;
 874
 875         // The archive to write to
 876         struct archive* archive;
 877
 878         // Resolver for hardlinks
 879         struct archive_entry_linkresolver* linkresolver;
 880
 881         // The filelist of all files to write
 882         struct pakfire_filelist* filelist;
 883
 884         // The progress indicator
 885         struct pakfire_progress* progress;
 886
 887         // Digests to write to the archive
 888         int digests;
 889 };
 890
 891 static int pakfire_copy_data_from_file(struct pakfire_ctx* ctx,
 892                 struct archive* archive, FILE* f) {
 893         char buffer[BUFFER_SIZE];
 894
 895         ssize_t bytes_read = 0;
 896         ssize_t bytes_written = 0;
 897
 898         // Read file from the very beginning - also allows calling this multiple times
 899         rewind(f);
 900
 901         // Loop through the entire length of the file
 902         while (!feof(f)) {
 903                 // Read a block from file
 904                 bytes_read = fread(buffer, 1, sizeof(buffer), f);
 905
 906                 // Check if any error occured
 907                 if (ferror(f)) {
 908                         CTX_ERROR(ctx, "Read error: %m\n");
 909                         return -errno;
 910                 }
 911
 912                 // Write the block to the archive
 913                 bytes_written = archive_write_data(archive, buffer, bytes_read);
 914                 if (bytes_written < bytes_read) {
 915                         CTX_ERROR(ctx, "Write error: %s\n", archive_error_string(archive));
 916                         return -errno;
 917                 }
 918         }
 919
 920         return 0;
 921 }
 922
 923 static int __pakfire_compress_entry(struct pakfire* pakfire, struct pakfire_file* file,
 924                 struct pakfire_compress* data, struct archive_entry* entry) {
 925         FILE* f = NULL;
 926         int r;
 927
 928         const char* path = archive_entry_pathname(entry);
 929
 930         // Remove any leading slahes
 931         while (*path == '/')
 932                 path++;
 933
 934         archive_entry_set_pathname(entry, path);
 935
 936         // Write the header
 937         r = archive_write_header(data->archive, entry);
 938         if (r) {
 939                 CTX_ERROR(data->ctx, "Error writing file header: %s\n",
 940                         archive_error_string(data->archive));
 941                 goto ERROR;
 942         }
 943
 944         // Copy the data if there is any
 945         if (archive_entry_size(entry)) {
 946                 // Open the file
 947                 f = pakfire_file_open(file);
 948                 if (!f) {
 949                         r = 1;
 950                         goto ERROR;
 951                 }
 952
 953                 // Copy the payload into the archive
 954                 r = pakfire_copy_data_from_file(data->ctx, data->archive, f);
 955                 if (r)
 956                         goto ERROR;
 957         }
 958
 959         // Write trailer
 960         r = archive_write_finish_entry(data->archive);
 961         if (r)
 962                 goto ERROR;
 963
 964 ERROR:
 965         if (f)
 966                 fclose(f);
 967
 968         return r;
 969 }
 970
 971 static int __pakfire_compress(struct pakfire* pakfire, struct pakfire_file* file, void* p) {
 972         struct archive_entry* entry = NULL;
 973         struct archive_entry* sparse_entry = NULL;
 974         int r = 1;
 975
 976         struct pakfire_compress* data = (struct pakfire_compress*)p;
 977
 978         // Fetch the file size
 979         const size_t size = pakfire_file_get_size(file);
 980
 981         // Generate file metadata into an archive entry
 982         entry = pakfire_file_archive_entry(file, data->digests);
 983         if (!entry) {
 984                 r = 1;
 985                 goto ERROR;
 986         }
 987
 988         // Perform search for hardlinks
 989         archive_entry_linkify(data->linkresolver, &entry, &sparse_entry);
 990
 991         // Write the main entry
 992         if (entry) {
 993                 r = __pakfire_compress_entry(pakfire, file, data, entry);
 994                 if (r)
 995                         goto ERROR;
 996         }
 997
 998         // Write the sparse entry
 999         if (sparse_entry) {
1000                 r = __pakfire_compress_entry(pakfire, file, data, sparse_entry);
1001                 if (r)
1002                         goto ERROR;
1003         }
1004
1005         // Query the link resolver for any more entries
1006         for (;;) {
1007                 // Free the entry
1008                 if (entry) {
1009                         archive_entry_free(entry);
1010                         entry = NULL;
1011                 }
1012
1013                 // Free the sparse entry
1014                 if (sparse_entry) {
1015                         archive_entry_free(sparse_entry);
1016                         sparse_entry = NULL;
1017                 }
1018
1019                 // Fetch the next entry
1020                 archive_entry_linkify(data->linkresolver, &entry, &sparse_entry);
1021                 if (!entry)
1022                         break;
1023
1024                 // Write the entry to the archive
1025                 r = __pakfire_compress_entry(pakfire, file, data, entry);
1026                 if (r)
1027                         goto ERROR;
1028         }
1029
1030         // Update the progress
1031         pakfire_progress_increment(data->progress, size);
1032
1033 ERROR:
1034         if (entry)
1035                 archive_entry_free(entry);
1036         if (sparse_entry)
1037                 archive_entry_free(sparse_entry);
1038
1039         return r;
1040 }
1041
1042 int pakfire_compress(struct pakfire* pakfire, struct archive* archive,
1043                 struct pakfire_filelist* filelist, const char* message, int flags, int digests) {
1044         int progress_flags =
1045                 PAKFIRE_PROGRESS_SHOW_PERCENTAGE |
1046                 PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED;
1047         int r = 1;
1048
1049         struct pakfire_ctx* ctx = pakfire_ctx(pakfire);
1050
1051         struct pakfire_compress data = {
1052                 .ctx      = ctx,
1053                 .pakfire  = pakfire,
1054                 .archive  = archive,
1055                 .filelist = filelist,
1056                 .flags    = flags,
1057                 .digests  = digests,
1058         };
1059
1060         // Should we show a progress bar?
1061         if (flags & PAKFIRE_COMPRESS_NO_PROGRESS)
1062                 progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
1063
1064         // Should we show throughput?
1065         if (flags & PAKFIRE_COMPRESS_SHOW_THROUGHPUT)
1066                 progress_flags |= PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED;
1067
1068         // Fetch the length of the filelist
1069         const size_t size = pakfire_filelist_total_size(filelist);
1070
1071         // Create the progress indicator
1072         r = pakfire_progress_create(&data.progress, ctx, progress_flags, NULL);
1073         if (r)
1074                 goto ERROR;
1075
1076         // Set progress title
1077         r = pakfire_progress_set_title(data.progress, "%s", message);
1078         if (r)
1079                 goto ERROR;
1080
1081         // Start progress
1082         r = pakfire_progress_start(data.progress, size);
1083         if (r)
1084                 goto ERROR;
1085
1086         // Setup the link resolver
1087         data.linkresolver = archive_entry_linkresolver_new();
1088         if (!data.linkresolver) {
1089                 CTX_ERROR(ctx, "Could not setup link resolver: m\n");
1090                 goto ERROR;
1091         }
1092
1093         // Set the link resolver strategy
1094         archive_entry_linkresolver_set_strategy(data.linkresolver, archive_format(archive));
1095
1096         // Walk through the entire filelist
1097         r = pakfire_filelist_walk(filelist, __pakfire_compress, &data, 0);
1098         if (r)
1099                 goto ERROR;
1100
1101         // Finish the progress
1102         if (data.progress)
1103                 pakfire_progress_finish(data.progress);
1104
1105 ERROR:
1106         if (data.progress)
1107                 pakfire_progress_unref(data.progress);
1108         if (data.linkresolver)
1109                 archive_entry_linkresolver_free(data.linkresolver);
1110         if (ctx)
1111                 pakfire_ctx_unref(ctx);
1112
1113         return r;
1114 }
1115
1116 int pakfire_compress_create_archive(struct pakfire* pakfire, struct archive** archive,
1117                 FILE* f, const enum pakfire_compressions compression, const unsigned int level) {
1118         struct archive* a = NULL;
1119         char value[16];
1120         int r;
1121
1122         struct pakfire_ctx* ctx = pakfire_ctx(pakfire);
1123
1124         // Open a new archive
1125         a = archive_write_new();
1126         if (!a) {
1127                 CTX_ERROR(ctx, "archive_write_new() failed\n");
1128                 r = 1;
1129                 goto ERROR;
1130         }
1131
1132         // Use the PAX format
1133         r = archive_write_set_format_pax(a);
1134         if (r) {
1135                 CTX_ERROR(ctx, "Could not set format to PAX: %s\n", archive_error_string(a));
1136                 goto ERROR;
1137         }
1138
1139         // Store any extended attributes in the SCHILY headers
1140         r = archive_write_set_format_option(a, "pax", "xattrheader", "SCHILY");
1141         if (r) {
1142                 CTX_ERROR(ctx, "Could not set xattrheader option: %s\n", archive_error_string(a));
1143                 goto ERROR;
1144         }
1145
1146         switch (compression) {
1147                 case PAKFIRE_COMPRESS_ZSTD:
1148                         // Enable Zstd
1149                         r = archive_write_add_filter_zstd(a);
1150                         if (r) {
1151                                 CTX_ERROR(ctx, "Could not enable Zstandard compression: %s\n",
1152                                         archive_error_string(a));
1153                                 goto ERROR;
1154                         }
1155
1156                         // Do not pad the last block
1157                         archive_write_set_bytes_in_last_block(a, 1);
1158
1159                         // Set compression level
1160                         if (level) {
1161                                 r = pakfire_string_format(value, "%u", level);
1162                                 if (r)
1163                                         goto ERROR;
1164
1165                                 r = archive_write_set_filter_option(a, NULL, "compression-level", value);
1166                                 if (r) {
1167                                         CTX_ERROR(ctx, "Could not set Zstandard compression level: %s\n",
1168                                                 archive_error_string(a));
1169                                         goto ERROR;
1170                                 }
1171                         }
1172
1173 #if ARCHIVE_VERSION_NUMBER >= 3006000
1174                         // Fetch numbers of processors
1175                         long processors = sysconf(_SC_NPROCESSORS_ONLN);
1176
1177                         if (processors > 1) {
1178                                 r = pakfire_string_format(value, "%ld", processors);
1179                                 if (r) {
1180                                         CTX_ERROR(ctx, "Could not format threads: %m\n");
1181                                         goto ERROR;
1182                                 }
1183
1184                                 // Try using multiple threads
1185                                 r = archive_write_set_filter_option(a, NULL, "threads", value);
1186                                 if (r) {
1187                                         CTX_ERROR(ctx, "Could not enable %ld threads for compression: %s\n",
1188                                                 processors, archive_error_string(a));
1189                                         goto ERROR;
1190                                 }
1191                         }
1192 #endif
1193         }
1194
1195         // Write archive to f
1196         if (f) {
1197                 r = archive_write_open_FILE(a, f);
1198                 if (r) {
1199                         CTX_ERROR(ctx, "archive_write_open_FILE() failed: %s\n", archive_error_string(a));
1200                         goto ERROR;
1201                 }
1202         }
1203
1204         // Success
1205         *archive = a;
1206
1207         if (ctx)
1208                 pakfire_ctx_unref(ctx);
1209
1210         return 0;
1211
1212 ERROR:
1213         if (a)
1214                 archive_write_free(a);
1215
1216         if (ctx)
1217                 pakfire_ctx_unref(ctx);
1218
1219         return r;
1220 }