src/libpakfire/compress.c

   1 /*#############################################################################
   2 #                                                                             #
   3 # Pakfire - The IPFire package management system                              #
   4 # Copyright (C) 2021 Pakfire development team                                 #
   5 #                                                                             #
   6 # This program is free software: you can redistribute it and/or modify        #
   7 # it under the terms of the GNU General Public License as published by        #
   8 # the Free Software Foundation, either version 3 of the License, or           #
   9 # (at your option) any later version.                                         #
  10 #                                                                             #
  11 # This program is distributed in the hope that it will be useful,             #
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  14 # GNU General Public License for more details.                                #
  15 #                                                                             #
  16 # You should have received a copy of the GNU General Public License           #
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  18 #                                                                             #
  19 #############################################################################*/
  20
  21 #include <errno.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25
  26 #include <archive.h>
  27 #include <lzma.h>
  28 #include <zstd.h>
  29
  30 // Enable legacy logging
  31 #define PAKFIRE_LEGACY_LOGGING
  32
  33 #include <pakfire/ctx.h>
  34 #include <pakfire/compress.h>
  35 #include <pakfire/file.h>
  36 #include <pakfire/filelist.h>
  37 #include <pakfire/logging.h>
  38 #include <pakfire/path.h>
  39 #include <pakfire/progress.h>
  40 #include <pakfire/string.h>
  41 #include <pakfire/util.h>
  42
  43 // Read up to N bytes for analyze the magic
  44 #define MAX_MAGIC_LENGTH        6
  45
  46 // Compression/Decompression buffer size
  47 #define BUFFER_SIZE                     64 * 1024
  48
  49 // Settings for XZ
  50 #define XZ_COMPRESSION_LEVEL    6
  51
  52 // Settings for ZSTD
  53 #define ZSTD_COMPRESSION_LEVEL  7
  54
  55 const struct compressor {
  56         char magic[MAX_MAGIC_LENGTH];
  57         size_t magic_length;
  58         FILE* (*open)(FILE* f, const char* mode);
  59 } compressors[] = {
  60         // XZ
  61         { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
  62         // ZSTD
  63         { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
  64         // End
  65         { "", 0, NULL, },
  66 };
  67
  68 // Try to guess the compression
  69 FILE* pakfire_xfopen(FILE* f, const char* mode) {
  70         char buffer[MAX_MAGIC_LENGTH];
  71
  72         if (!f) {
  73                 errno = EBADFD;
  74                 return NULL;
  75         }
  76
  77         if (!mode) {
  78                 errno = EINVAL;
  79                 return NULL;
  80         }
  81
  82         // This only works for reading files
  83         if (*mode != 'r') {
  84                 errno = ENOTSUP;
  85                 return NULL;
  86         }
  87
  88         fpos_t pos;
  89
  90         // Store the position
  91         int r = fgetpos(f, &pos);
  92         if (r < 0)
  93                 return NULL;
  94
  95         // Read a couple of bytes
  96         size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
  97
  98         // Reset position
  99         r = fsetpos(f, &pos);
 100         if (r < 0)
 101                 return NULL;
 102
 103         // Check if we could read anything
 104         if (!bytes_read || bytes_read < sizeof(buffer))
 105                 return f;
 106
 107         // Analyze magic
 108         for (const struct compressor* c = compressors; c->open; c++) {
 109                 // Check if we have read enough data
 110                 if (bytes_read < c->magic_length)
 111                         continue;
 112
 113                 // Compare the magic value
 114                 r = memcmp(c->magic, buffer, c->magic_length);
 115                 if (r)
 116                         continue;
 117
 118                 // We found a match!
 119                 return c->open(f, mode);
 120         }
 121
 122         // Nothing seems to match
 123         errno = ENOTSUP;
 124         return f;
 125 }
 126
 127 struct xz_cookie {
 128         FILE* f;
 129         char mode;
 130         lzma_stream stream;
 131         int done;
 132
 133         uint8_t buffer[BUFFER_SIZE];
 134 };
 135
 136 static ssize_t xz_read(void* data, char* buffer, size_t size) {
 137         struct xz_cookie* cookie = (struct xz_cookie*)data;
 138         if (!cookie)
 139                 return -1;
 140
 141         // Do not read when mode is "w"
 142         if (cookie->mode == 'w')
 143                 return -1;
 144
 145         lzma_action action = LZMA_RUN;
 146
 147         // Set output to allocated buffer
 148         cookie->stream.next_out  = (uint8_t *)buffer;
 149         cookie->stream.avail_out = size;
 150
 151         while (1) {
 152                 // Read something when the input buffer is empty
 153                 if (cookie->stream.avail_in == 0) {
 154                         cookie->stream.next_in  = cookie->buffer;
 155                         cookie->stream.avail_in = fread(cookie->buffer,
 156                                 1, sizeof(cookie->buffer), cookie->f);
 157
 158                         // Break if the input file could not be read
 159                         if (ferror(cookie->f))
 160                                 return -1;
 161
 162                         // Finish after we have reached the end of the input file
 163                         if (feof(cookie->f))
 164                                 cookie->done = 1;
 165                 }
 166
 167                 lzma_ret ret = lzma_code(&cookie->stream, action);
 168
 169                 // If the stream has ended, we just send the
 170                 // remaining output and mark that we are done.
 171                 if (ret == LZMA_STREAM_END) {
 172                         cookie->done = 1;
 173                         return size - cookie->stream.avail_out;
 174                 }
 175
 176                 // Break on all other unexpected errors
 177                 if (ret != LZMA_OK)
 178                         return -1;
 179
 180                 // When we have read enough to fill the entire output buffer, we return
 181                 if (cookie->stream.avail_out == 0)
 182                         return size;
 183
 184                 if (cookie->done)
 185                         return -1;
 186         }
 187 }
 188
 189 static ssize_t xz_write(void* data, const char* buffer, size_t size) {
 190         struct xz_cookie* cookie = (struct xz_cookie*)data;
 191         if (!cookie)
 192                 return -1;
 193
 194         // Do not write when mode is "r"
 195         if (cookie->mode == 'r')
 196                 return -1;
 197
 198         // Return nothing when there is no input
 199         if (size == 0)
 200                 return 0;
 201
 202         // Set input to allocated buffer
 203         cookie->stream.next_in  = (uint8_t *)buffer;
 204         cookie->stream.avail_in = size;
 205
 206         while (1) {
 207                 cookie->stream.next_out  = cookie->buffer;
 208                 cookie->stream.avail_out = sizeof(cookie->buffer);
 209
 210                 lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
 211                 if (ret != LZMA_OK)
 212                         return -1;
 213
 214                 size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 215                 if (bytes_to_write) {
 216                         size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 217
 218                         if (bytes_written != bytes_to_write)
 219                                 return -1;
 220                 }
 221
 222                 // Report that all data has been written
 223                 if (cookie->stream.avail_in == 0)
 224                         return size;
 225         }
 226 }
 227
 228 static int xz_close(void* data) {
 229         struct xz_cookie* cookie = (struct xz_cookie*)data;
 230         if (!cookie)
 231                 return -1;
 232
 233         if (cookie->mode == 'w') {
 234                 while (1) {
 235                         cookie->stream.next_out  = cookie->buffer;
 236                         cookie->stream.avail_out = sizeof(cookie->buffer);
 237
 238                         lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
 239                         if (ret != LZMA_OK && ret != LZMA_STREAM_END)
 240                                 return -1;
 241
 242                         size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 243                         if (bytes_to_write) {
 244                                 size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 245
 246                                 if (bytes_written != bytes_to_write)
 247                                         return -1;
 248                         }
 249
 250                         if (ret == LZMA_STREAM_END)
 251                                 break;
 252                 }
 253         }
 254
 255         lzma_end(&cookie->stream);
 256
 257         // Close input file
 258         int r = fclose(cookie->f);
 259         free(cookie);
 260
 261         return r;
 262 }
 263
 264 static cookie_io_functions_t xz_functions = {
 265         .read  = xz_read,
 266         .write = xz_write,
 267         .seek  = NULL,
 268         .close = xz_close,
 269 };
 270
 271 FILE* pakfire_xzfopen(FILE* f, const char* mode) {
 272         lzma_ret ret;
 273
 274         if (!f) {
 275                 errno = EBADFD;
 276                 return NULL;
 277         }
 278
 279         if (!mode) {
 280                 errno = EINVAL;
 281                 return NULL;
 282         }
 283
 284         struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
 285         if (!cookie)
 286                 return NULL;
 287
 288         cookie->f = f;
 289         cookie->mode = *mode;
 290
 291         switch (cookie->mode) {
 292                 case 'r':
 293                         ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
 294                         break;
 295
 296                 case 'w':
 297                         ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
 298                         break;
 299
 300                 default:
 301                         errno = ENOTSUP;
 302                         goto ERROR;
 303         }
 304
 305         if (ret != LZMA_OK)
 306                 goto ERROR;
 307
 308         return fopencookie(cookie, mode, xz_functions);
 309
 310 ERROR:
 311         free(cookie);
 312         return NULL;
 313 }
 314
 315 // ZSTD
 316
 317 struct zstd_cookie {
 318         FILE* f;
 319         char mode;
 320         int done;
 321
 322         // Encoder
 323         ZSTD_CStream* cstream;
 324         ZSTD_inBuffer in;
 325
 326         // Decoder
 327         ZSTD_DStream* dstream;
 328         ZSTD_outBuffer out;
 329
 330         uint8_t buffer[BUFFER_SIZE];
 331 };
 332
 333 static ssize_t zstd_read(void* data, char* buffer, size_t size) {
 334         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 335         if (!cookie)
 336                 return -1;
 337
 338         // Do not read when mode is "w"
 339         if (cookie->mode == 'w')
 340                 return -1;
 341
 342         if (cookie->done)
 343                 return 0;
 344
 345         size_t r = 0;
 346
 347         // Configure output buffer
 348         cookie->out.dst = buffer;
 349         cookie->out.pos = 0;
 350         cookie->out.size = size;
 351
 352         while (1) {
 353                 if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
 354                         cookie->in.pos = 0;
 355                         cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
 356                 }
 357
 358                 if (r || cookie->in.size)
 359                         r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
 360
 361                 if (r == 0 && feof(cookie->f)) {
 362                         cookie->done = 1;
 363                         return cookie->out.pos;
 364                 }
 365
 366                 if (ZSTD_isError(r))
 367                         return -1;
 368
 369                 // Buffer full
 370                 if (cookie->out.pos == size)
 371                         return size;
 372         }
 373 }
 374
 375 static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
 376         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 377         if (!cookie)
 378                 return -1;
 379
 380         // Do not write when mode is "r"
 381         if (cookie->mode == 'r')
 382                 return -1;
 383
 384         // Return nothing when there is no input
 385         if (size == 0)
 386                 return 0;
 387
 388         // Configure input buffer
 389         cookie->in.src = buffer;
 390         cookie->in.pos = 0;
 391         cookie->in.size = size;
 392
 393         while (1) {
 394                 cookie->out.pos = 0;
 395
 396                 size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
 397                 if (ZSTD_isError(r))
 398                         return -1;
 399
 400                 if (cookie->out.pos > 0) {
 401                         size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 402
 403                         if (bytes_written != cookie->out.pos)
 404                                 return -1;
 405                 }
 406
 407                 // Return when all input has been written
 408                 if (cookie->in.pos == size)
 409                         return size;
 410         }
 411 }
 412
 413 static int zstd_close(void* data) {
 414         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 415         if (!cookie)
 416                 return -1;
 417
 418         if (cookie->mode == 'w') {
 419                 while (1) {
 420                         // Reset output buffer
 421                         cookie->out.pos = 0;
 422
 423                         size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
 424                         if (ZSTD_isError(r))
 425                                 return -1;
 426
 427                         if (cookie->out.pos > 0) {
 428                                 size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 429
 430                                 if (bytes_written != cookie->out.pos)
 431                                         return -1;
 432                         }
 433
 434                         if (r == 0)
 435                                 break;
 436                 }
 437         }
 438
 439         int r = fclose(cookie->f);
 440
 441         // Free everything
 442         if (cookie->cstream)
 443                 ZSTD_freeCStream(cookie->cstream);
 444         if (cookie->dstream)
 445                 ZSTD_freeDStream(cookie->dstream);
 446         free(cookie);
 447
 448         return r;
 449 }
 450
 451 static cookie_io_functions_t zstd_functions = {
 452         .read  = zstd_read,
 453         .write = zstd_write,
 454         .seek  = NULL,
 455         .close = zstd_close,
 456 };
 457
 458 FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
 459         if (!f) {
 460                 errno = EBADFD;
 461                 return NULL;
 462         }
 463
 464         if (!mode) {
 465                 errno = EINVAL;
 466                 return NULL;
 467         }
 468
 469         struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
 470         if (!cookie)
 471                 return NULL;
 472
 473         cookie->f = f;
 474         cookie->mode = *mode;
 475
 476         size_t r;
 477         switch (cookie->mode) {
 478                 case 'r':
 479                         // Allocate stream
 480                         cookie->dstream = ZSTD_createDStream();
 481                         if (!cookie->dstream)
 482                                 goto ERROR;
 483
 484                         // Initialize stream
 485                         r = ZSTD_initDStream(cookie->dstream);
 486                         if (ZSTD_isError(r))
 487                                 goto ERROR;
 488
 489                         cookie->in.src = cookie->buffer;
 490                         cookie->in.pos = 0;
 491                         cookie->in.size = 0;
 492                         break;
 493
 494                 case 'w':
 495                         // Allocate stream
 496                         cookie->cstream = ZSTD_createCStream();
 497                         if (!cookie->cstream)
 498                                 goto ERROR;
 499
 500                         // Initialize stream
 501                         r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
 502                         if (ZSTD_isError(r))
 503                                 goto ERROR;
 504
 505                         cookie->out.dst = cookie->buffer;
 506                         cookie->out.pos = 0;
 507                         cookie->out.size = sizeof(cookie->buffer);
 508                         break;
 509
 510                 default:
 511                         errno = ENOTSUP;
 512                         goto ERROR;
 513         }
 514
 515         return fopencookie(cookie, mode, zstd_functions);
 516
 517 ERROR:
 518         if (cookie->cstream)
 519                 ZSTD_freeCStream(cookie->cstream);
 520         if (cookie->dstream)
 521                 ZSTD_freeDStream(cookie->dstream);
 522         free(cookie);
 523
 524         return NULL;
 525 }
 526
 527 /*
 528         Helper function to conditionally walk through an archive
 529         and perform actions based on the callback.
 530 */
 531 int pakfire_walk(struct pakfire_ctx* ctx, struct archive* archive,
 532                 pakfire_walk_callback callback, pakfire_walk_filter_callback filter_callback,
 533                 void* p) {
 534         struct archive_entry* entry = NULL;
 535         int r;
 536
 537         // Walk through the archive
 538         for (;;) {
 539                 r = archive_read_next_header(archive, &entry);
 540
 541                 // Handle the return code
 542                 switch (r) {
 543                         // Fall through if everything is okay
 544                         case ARCHIVE_OK:
 545                                 break;
 546
 547                         // Return OK when we reached the end of the archive
 548                         case ARCHIVE_EOF:
 549                                 return 0;
 550
 551                         // Raise any other errors
 552                         default:
 553                                 return r;
 554                 }
 555
 556                 // Call the filter callback before we call the actual callback
 557                 if (filter_callback) {
 558                         r = filter_callback(ctx, archive, entry, p);
 559
 560                         // Handle the return code
 561                         switch (r) {
 562                                 case PAKFIRE_WALK_OK:
 563                                         break;
 564
 565                                 case PAKFIRE_WALK_END:
 566                                         CTX_DEBUG(ctx, "Filter callback sent END\n");
 567                                         return 0;
 568
 569                                 case PAKFIRE_WALK_SKIP:
 570                                         CTX_DEBUG(ctx, "Filter callback sent SKIP\n");
 571                                         continue;
 572
 573                                 case PAKFIRE_WALK_DONE:
 574                                         CTX_DEBUG(ctx, "Filter callback sent DONE\n");
 575
 576                                         // Clear the callback function
 577                                         filter_callback = NULL;
 578                                         break;
 579
 580                                 case PAKFIRE_WALK_AGAIN:
 581                                         CTX_DEBUG(ctx, "Filter callback sent AGAIN\n");
 582                                         return -EAGAIN;
 583
 584                                 // Raise any other errors
 585                                 default:
 586                                         CTX_DEBUG(ctx, "Filter callback returned an error: %d\n", r);
 587                                         return r;
 588                         }
 589                 }
 590
 591                 // Run callback
 592                 if (callback) {
 593                         r = callback(ctx, archive, entry, p);
 594
 595                         // Handle the return code
 596                         switch (r) {
 597                                 case PAKFIRE_WALK_OK:
 598                                         break;
 599
 600                                 case PAKFIRE_WALK_DONE:
 601                                         CTX_DEBUG(ctx, "Callback sent DONE\n");
 602                                         return 0;
 603
 604                                 // Raise any other errors
 605                                 default:
 606                                         return r;
 607                         }
 608                 }
 609         }
 610
 611         return 0;
 612 }
 613
 614 // Common extraction
 615
 616 struct pakfire_extract {
 617         // Reference to Pakfire
 618         struct pakfire* pakfire;
 619
 620         // Flags
 621         int flags;
 622
 623         // The archive to extract
 624         struct archive* archive;
 625
 626         // The filelist of all extracted files
 627         struct pakfire_filelist* filelist;
 628
 629         // Prepend this prefix
 630         const char* prefix;
 631
 632         // The writer
 633         struct archive* writer;
 634
 635         // The progress indicator
 636         struct pakfire_progress* progress;
 637 };
 638
 639 static void pakfire_extract_progress(void* p) {
 640         struct pakfire_extract* data = (struct pakfire_extract*)p;
 641
 642         // Fetch how many bytes have been read
 643         const size_t position = archive_filter_bytes(data->archive, -1);
 644
 645         // Update progress
 646         pakfire_progress_update(data->progress, position);
 647 }
 648
 649 static int __pakfire_extract(struct pakfire_ctx* ctx, struct archive* a,
 650                 struct archive_entry* entry, void* p) {
 651         struct pakfire_file* file = NULL;
 652         struct vfs_cap_data cap_data = {};
 653         char buffer[PATH_MAX];
 654         int r;
 655
 656         struct pakfire_extract* data = (struct pakfire_extract*)p;
 657
 658         // Fetch path
 659         const char* path = archive_entry_pathname(entry);
 660
 661         // Make sure we have a leading slash on the filelist
 662         if (!pakfire_string_startswith(path, "/")) {
 663                 r = pakfire_string_format(buffer, "/%s", path);
 664                 if (r)
 665                         goto ERROR;
 666
 667                 // Store the new name
 668                 archive_entry_set_pathname(entry, buffer);
 669
 670                 // Update the path pointer
 671                 path = archive_entry_pathname(entry);
 672         }
 673
 674         // Generate a file object
 675         r = pakfire_file_create_from_archive_entry(&file, data->pakfire, entry);
 676         if (r)
 677                 goto ERROR;
 678
 679         // Add entry to filelist (if requested)
 680         if (data->filelist) {
 681                 // Append the file to the list
 682                 r = pakfire_filelist_add(data->filelist, file);
 683                 if (r)
 684                         goto ERROR;
 685         }
 686
 687         const int configfile = pakfire_file_has_flag(file, PAKFIRE_FILE_CONFIG);
 688
 689         // Prepend the prefix
 690         if (*data->prefix) {
 691                 // Compose file path
 692                 r = pakfire_path_append(buffer, data->prefix, path);
 693                 if (r) {
 694                         CTX_ERROR(ctx, "Could not compose file path: %m\n");
 695                         goto ERROR;
 696                 }
 697
 698                 // Set file path
 699                 archive_entry_set_pathname(entry, buffer);
 700
 701                 // Update hardlink destination
 702                 const char* link = archive_entry_hardlink(entry);
 703                 if (link) {
 704                         r = pakfire_path_append(buffer, data->prefix, link);
 705                         if (r) {
 706                                 CTX_ERROR(ctx, "Could not compose hardlink path: %m\n");
 707                                 goto ERROR;
 708                         }
 709
 710                         // Set hardlink path
 711                         archive_entry_set_hardlink(entry, buffer);
 712                 }
 713         }
 714
 715         if (configfile) {
 716                 // Fetch path again since we changed it
 717                 path = archive_entry_pathname(entry);
 718
 719                 if (pakfire_path_exists(path)) {
 720                         CTX_DEBUG(ctx, "The configuration file %s exists\n",
 721                                 pakfire_file_get_path(file));
 722
 723                         r = pakfire_string_format(buffer, "%s.paknew", path);
 724                         if (r) {
 725                                 CTX_ERROR(ctx, "Could not compose path for configuration file: %m\n");
 726                                 goto ERROR;
 727                         }
 728
 729                         // Set the path again
 730                         archive_entry_set_pathname(entry, buffer);
 731                 }
 732         }
 733
 734         // Create file & extract payload
 735         if (data->writer) {
 736                 // Fetch path again since we changed it
 737                 path = archive_entry_pathname(entry);
 738
 739                 CTX_DEBUG(ctx, "Extracting %s\n", path);
 740
 741                 // Remove any extended attributes which we never write to disk
 742                 archive_entry_xattr_clear(entry);
 743
 744                 // Set capabilities
 745                 if (pakfire_file_has_caps(file)) {
 746                         r = pakfire_file_write_fcaps(file, &cap_data);
 747                         if (r)
 748                                 goto ERROR;
 749
 750                         // Store capabilities in archive entry
 751                         archive_entry_xattr_add_entry(entry, "security.capability",
 752                                 &cap_data, sizeof(cap_data));
 753                 }
 754
 755                 // Write payload
 756                 r = archive_read_extract2(data->archive, entry, data->writer);
 757                 switch (r) {
 758                         case ARCHIVE_OK:
 759                                 r = 0;
 760                                 break;
 761
 762                         case ARCHIVE_WARN:
 763                                 CTX_ERROR(ctx, "%s\n", archive_error_string(data->writer));
 764
 765                                 // Pretend everything has been okay
 766                                 r = 0;
 767                                 break;
 768
 769                         case ARCHIVE_FATAL:
 770                                 CTX_ERROR(ctx, "%s\n", archive_error_string(data->writer));
 771                                 r = 1;
 772                                 break;
 773                 }
 774         }
 775
 776 ERROR:
 777         if (file)
 778                 pakfire_file_unref(file);
 779
 780         return r;
 781 }
 782
 783 int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
 784                 size_t size, struct pakfire_filelist* filelist,
 785                 const char* prefix, const char* message,
 786                 pakfire_walk_filter_callback filter_callback, int flags) {
 787         int progress_flags = PAKFIRE_PROGRESS_SHOW_PERCENTAGE;
 788         int r = 1;
 789
 790         struct pakfire_ctx* ctx = pakfire_ctx(pakfire);
 791
 792         // Use / if no prefix is set
 793         if (!prefix)
 794                 prefix = "/";
 795
 796         struct pakfire_extract data = {
 797                 .pakfire  = pakfire,
 798                 .archive  = archive,
 799                 .filelist = filelist,
 800                 .prefix   = prefix,
 801                 .flags    = flags,
 802                 .writer   = NULL,
 803         };
 804
 805         // Is this a dry run?
 806         const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;
 807
 808         // Allocate writer
 809         if (!dry_run) {
 810                 data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
 811                 if (!data.writer) {
 812                         ERROR(pakfire, "Could not create disk writer: %m\n");
 813                         r = 1;
 814                         goto ERROR;
 815                 }
 816         }
 817
 818         // Should we show any progress?
 819         if (flags & PAKFIRE_EXTRACT_NO_PROGRESS)
 820                 progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
 821
 822         // Show throughput?
 823         if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT)
 824                 progress_flags |= PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED;
 825
 826         // Create the progress indicator
 827         r = pakfire_progress_create(&data.progress, ctx, progress_flags, NULL);
 828         if (r)
 829                 goto ERROR;
 830
 831         // Set the title
 832         r = pakfire_progress_set_title(data.progress, "%s", message);
 833         if (r)
 834                 goto ERROR;
 835
 836         // Register progress callback
 837         archive_read_extract_set_progress_callback(data.archive,
 838                 pakfire_extract_progress, &data);
 839
 840         // Start progress
 841         r = pakfire_progress_start(data.progress, size);
 842         if (r)
 843                 goto ERROR;
 844
 845         // Walk through the entire archive
 846         r = pakfire_walk(ctx, archive, __pakfire_extract, filter_callback, &data);
 847         if (r)
 848                 goto ERROR;
 849
 850         // Finish the progress
 851         r = pakfire_progress_finish(data.progress);
 852         if (r)
 853                 goto ERROR;
 854
 855 ERROR:
 856         if (data.progress)
 857                 pakfire_progress_unref(data.progress);
 858         if (data.writer)
 859                 archive_write_free(data.writer);
 860         if (ctx)
 861                 pakfire_ctx_unref(ctx);
 862
 863         return r;
 864 }
 865
 866 // Common compression
 867
 868 struct pakfire_compress {
 869         // Reference to Pakfire
 870         struct pakfire* pakfire;
 871
 872         // Flags
 873         int flags;
 874
 875         // The archive to write to
 876         struct archive* archive;
 877
 878         // Resolver for hardlinks
 879         struct archive_entry_linkresolver* linkresolver;
 880
 881         // The filelist of all files to write
 882         struct pakfire_filelist* filelist;
 883
 884         // The progress indicator
 885         struct pakfire_progress* progress;
 886
 887         // Digests to write to the archive
 888         int digests;
 889 };
 890
 891 static int pakfire_copy_data_from_file(struct pakfire* pakfire,
 892                 struct archive* archive, FILE* f) {
 893         char buffer[BUFFER_SIZE];
 894
 895         ssize_t bytes_read = 0;
 896         ssize_t bytes_written = 0;
 897
 898         // Read file from the very beginning - also allows calling this multiple times
 899         rewind(f);
 900
 901         // Loop through the entire length of the file
 902         while (!feof(f)) {
 903                 // Read a block from file
 904                 bytes_read = fread(buffer, 1, sizeof(buffer), f);
 905
 906                 // Check if any error occured
 907                 if (ferror(f)) {
 908                         ERROR(pakfire, "Read error: %m\n");
 909                         return -errno;
 910                 }
 911
 912                 // Write the block to the archive
 913                 bytes_written = archive_write_data(archive, buffer, bytes_read);
 914                 if (bytes_written < bytes_read) {
 915                         ERROR(pakfire, "Write error: %s\n", archive_error_string(archive));
 916                         return -errno;
 917                 }
 918         }
 919
 920         return 0;
 921 }
 922
 923 static int __pakfire_compress_entry(struct pakfire* pakfire, struct pakfire_file* file,
 924                 struct pakfire_compress* data, struct archive_entry* entry) {
 925         FILE* f = NULL;
 926         int r;
 927
 928         const char* path = archive_entry_pathname(entry);
 929
 930         // Remove any leading slahes
 931         while (*path == '/')
 932                 path++;
 933
 934         archive_entry_set_pathname(entry, path);
 935
 936         // Write the header
 937         r = archive_write_header(data->archive, entry);
 938         if (r) {
 939                 ERROR(pakfire, "Error writing file header: %s\n",
 940                         archive_error_string(data->archive));
 941                 goto ERROR;
 942         }
 943
 944         // Copy the data if there is any
 945         if (archive_entry_size(entry)) {
 946                 // Open the file
 947                 f = pakfire_file_open(file);
 948                 if (!f) {
 949                         r = 1;
 950                         goto ERROR;
 951                 }
 952
 953                 // Copy the payload into the archive
 954                 r = pakfire_copy_data_from_file(pakfire, data->archive, f);
 955                 if (r)
 956                         goto ERROR;
 957         }
 958
 959         // Write trailer
 960         r = archive_write_finish_entry(data->archive);
 961         if (r)
 962                 goto ERROR;
 963
 964 ERROR:
 965         if (f)
 966                 fclose(f);
 967
 968         return r;
 969 }
 970
 971 static int __pakfire_compress(struct pakfire* pakfire, struct pakfire_file* file, void* p) {
 972         struct archive_entry* entry = NULL;
 973         struct archive_entry* sparse_entry = NULL;
 974         int r = 1;
 975
 976         struct pakfire_compress* data = (struct pakfire_compress*)p;
 977
 978         // Fetch the file size
 979         const size_t size = pakfire_file_get_size(file);
 980
 981         // Generate file metadata into an archive entry
 982         entry = pakfire_file_archive_entry(file, data->digests);
 983         if (!entry) {
 984                 r = 1;
 985                 goto ERROR;
 986         }
 987
 988         // Perform search for hardlinks
 989         archive_entry_linkify(data->linkresolver, &entry, &sparse_entry);
 990
 991         // Write the main entry
 992         if (entry) {
 993                 r = __pakfire_compress_entry(pakfire, file, data, entry);
 994                 if (r)
 995                         goto ERROR;
 996         }
 997
 998         // Write the sparse entry
 999         if (sparse_entry) {
1000                 r = __pakfire_compress_entry(pakfire, file, data, sparse_entry);
1001                 if (r)
1002                         goto ERROR;
1003         }
1004
1005         // Query the link resolver for any more entries
1006         for (;;) {
1007                 // Free the entry
1008                 if (entry) {
1009                         archive_entry_free(entry);
1010                         entry = NULL;
1011                 }
1012
1013                 // Free the sparse entry
1014                 if (sparse_entry) {
1015                         archive_entry_free(sparse_entry);
1016                         sparse_entry = NULL;
1017                 }
1018
1019                 // Fetch the next entry
1020                 archive_entry_linkify(data->linkresolver, &entry, &sparse_entry);
1021                 if (!entry)
1022                         break;
1023
1024                 // Write the entry to the archive
1025                 r = __pakfire_compress_entry(pakfire, file, data, entry);
1026                 if (r)
1027                         goto ERROR;
1028         }
1029
1030         // Update the progress
1031         pakfire_progress_increment(data->progress, size);
1032
1033 ERROR:
1034         if (entry)
1035                 archive_entry_free(entry);
1036         if (sparse_entry)
1037                 archive_entry_free(sparse_entry);
1038
1039         return r;
1040 }
1041
1042 int pakfire_compress(struct pakfire* pakfire, struct archive* archive,
1043                 struct pakfire_filelist* filelist, const char* message, int flags, int digests) {
1044         int progress_flags =
1045                 PAKFIRE_PROGRESS_SHOW_PERCENTAGE |
1046                 PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED;
1047         int r = 1;
1048
1049         struct pakfire_ctx* ctx = pakfire_ctx(pakfire);
1050
1051         struct pakfire_compress data = {
1052                 .pakfire  = pakfire,
1053                 .archive  = archive,
1054                 .filelist = filelist,
1055                 .flags    = flags,
1056                 .digests  = digests,
1057         };
1058
1059         // Should we show a progress bar?
1060         if (flags & PAKFIRE_COMPRESS_NO_PROGRESS)
1061                 progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
1062
1063         // Should we show throughput?
1064         if (flags & PAKFIRE_COMPRESS_SHOW_THROUGHPUT)
1065                 progress_flags |= PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED;
1066
1067         // Fetch the length of the filelist
1068         const size_t size = pakfire_filelist_total_size(filelist);
1069
1070         // Create the progress indicator
1071         r = pakfire_progress_create(&data.progress, ctx, progress_flags, NULL);
1072         if (r)
1073                 goto ERROR;
1074
1075         // Set progress title
1076         r = pakfire_progress_set_title(data.progress, "%s", message);
1077         if (r)
1078                 goto ERROR;
1079
1080         // Start progress
1081         r = pakfire_progress_start(data.progress, size);
1082         if (r)
1083                 goto ERROR;
1084
1085         // Setup the link resolver
1086         data.linkresolver = archive_entry_linkresolver_new();
1087         if (!data.linkresolver) {
1088                 ERROR(pakfire, "Could not setup link resolver: m\n");
1089                 goto ERROR;
1090         }
1091
1092         // Set the link resolver strategy
1093         archive_entry_linkresolver_set_strategy(data.linkresolver, archive_format(archive));
1094
1095         // Walk through the entire filelist
1096         r = pakfire_filelist_walk(filelist, __pakfire_compress, &data, 0);
1097         if (r)
1098                 goto ERROR;
1099
1100         // Finish the progress
1101         if (data.progress)
1102                 pakfire_progress_finish(data.progress);
1103
1104 ERROR:
1105         if (data.progress)
1106                 pakfire_progress_unref(data.progress);
1107         if (data.linkresolver)
1108                 archive_entry_linkresolver_free(data.linkresolver);
1109         if (ctx)
1110                 pakfire_ctx_unref(ctx);
1111
1112         return r;
1113 }
1114
1115 int pakfire_compress_create_archive(struct pakfire* pakfire, struct archive** archive,
1116                 FILE* f, const enum pakfire_compressions compression, const unsigned int level) {
1117         struct archive* a = NULL;
1118         char value[16];
1119         int r;
1120
1121         // Open a new archive
1122         a = archive_write_new();
1123         if (!a) {
1124                 ERROR(pakfire, "archive_write_new() failed\n");
1125                 r = 1;
1126                 goto ERROR;
1127         }
1128
1129         // Use the PAX format
1130         r = archive_write_set_format_pax(a);
1131         if (r) {
1132                 ERROR(pakfire, "Could not set format to PAX: %s\n", archive_error_string(a));
1133                 goto ERROR;
1134         }
1135
1136         // Store any extended attributes in the SCHILY headers
1137         r = archive_write_set_format_option(a, "pax", "xattrheader", "SCHILY");
1138         if (r) {
1139                 ERROR(pakfire, "Could not set xattrheader option: %s\n", archive_error_string(a));
1140                 goto ERROR;
1141         }
1142
1143         switch (compression) {
1144                 case PAKFIRE_COMPRESS_ZSTD:
1145                         // Enable Zstd
1146                         r = archive_write_add_filter_zstd(a);
1147                         if (r) {
1148                                 ERROR(pakfire, "Could not enable Zstandard compression: %s\n",
1149                                         archive_error_string(a));
1150                                 goto ERROR;
1151                         }
1152
1153                         // Do not pad the last block
1154                         archive_write_set_bytes_in_last_block(a, 1);
1155
1156                         // Set compression level
1157                         if (level) {
1158                                 r = pakfire_string_format(value, "%u", level);
1159                                 if (r)
1160                                         goto ERROR;
1161
1162                                 r = archive_write_set_filter_option(a, NULL, "compression-level", value);
1163                                 if (r) {
1164                                         ERROR(pakfire, "Could not set Zstandard compression level: %s\n",
1165                                                 archive_error_string(a));
1166                                         goto ERROR;
1167                                 }
1168                         }
1169
1170 #if ARCHIVE_VERSION_NUMBER >= 3006000
1171                         // Fetch numbers of processors
1172                         long processors = sysconf(_SC_NPROCESSORS_ONLN);
1173
1174                         if (processors > 1) {
1175                                 r = pakfire_string_format(value, "%ld", processors);
1176                                 if (r) {
1177                                         ERROR(pakfire, "Could not format threads: %m\n");
1178                                         goto ERROR;
1179                                 }
1180
1181                                 // Try using multiple threads
1182                                 r = archive_write_set_filter_option(a, NULL, "threads", value);
1183                                 if (r) {
1184                                         ERROR(pakfire, "Could not enable %ld threads for compression: %s\n",
1185                                                 processors, archive_error_string(a));
1186                                         goto ERROR;
1187                                 }
1188                         }
1189 #endif
1190         }
1191
1192         // Write archive to f
1193         if (f) {
1194                 r = archive_write_open_FILE(a, f);
1195                 if (r) {
1196                         ERROR(pakfire, "archive_write_open_FILE() failed: %s\n", archive_error_string(a));
1197                         goto ERROR;
1198                 }
1199         }
1200
1201         // Success
1202         *archive = a;
1203
1204         return 0;
1205
1206 ERROR:
1207         if (a)
1208                 archive_write_free(a);
1209
1210         return r;
1211 }