src/libpakfire/compress.c

   1 /*#############################################################################
   2 #                                                                             #
   3 # Pakfire - The IPFire package management system                              #
   4 # Copyright (C) 2021 Pakfire development team                                 #
   5 #                                                                             #
   6 # This program is free software: you can redistribute it and/or modify        #
   7 # it under the terms of the GNU General Public License as published by        #
   8 # the Free Software Foundation, either version 3 of the License, or           #
   9 # (at your option) any later version.                                         #
  10 #                                                                             #
  11 # This program is distributed in the hope that it will be useful,             #
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  14 # GNU General Public License for more details.                                #
  15 #                                                                             #
  16 # You should have received a copy of the GNU General Public License           #
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  18 #                                                                             #
  19 #############################################################################*/
  20
  21 #include <errno.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25
  26 #include <archive.h>
  27 #include <lzma.h>
  28 #include <zstd.h>
  29
  30 #include <pakfire/ctx.h>
  31 #include <pakfire/compress.h>
  32 #include <pakfire/file.h>
  33 #include <pakfire/filelist.h>
  34 #include <pakfire/logging.h>
  35 #include <pakfire/path.h>
  36 #include <pakfire/progress.h>
  37 #include <pakfire/string.h>
  38 #include <pakfire/util.h>
  39
  40 // Read up to N bytes for analyze the magic
  41 #define MAX_MAGIC_LENGTH        6
  42
  43 // Compression/Decompression buffer size
  44 #define BUFFER_SIZE                     64 * 1024
  45
  46 // Settings for XZ
  47 #define XZ_COMPRESSION_LEVEL    6
  48
  49 // Settings for ZSTD
  50 #define ZSTD_COMPRESSION_LEVEL  7
  51
  52 const struct compressor {
  53         char magic[MAX_MAGIC_LENGTH];
  54         size_t magic_length;
  55         FILE* (*open)(FILE* f, const char* mode);
  56 } compressors[] = {
  57         // XZ
  58         { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
  59         // ZSTD
  60         { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
  61         // End
  62         { "", 0, NULL, },
  63 };
  64
  65 // Try to guess the compression
  66 FILE* pakfire_xfopen(FILE* f, const char* mode) {
  67         char buffer[MAX_MAGIC_LENGTH];
  68
  69         if (!f) {
  70                 errno = EBADFD;
  71                 return NULL;
  72         }
  73
  74         if (!mode) {
  75                 errno = EINVAL;
  76                 return NULL;
  77         }
  78
  79         // This only works for reading files
  80         if (*mode != 'r') {
  81                 errno = ENOTSUP;
  82                 return NULL;
  83         }
  84
  85         fpos_t pos;
  86
  87         // Store the position
  88         int r = fgetpos(f, &pos);
  89         if (r < 0)
  90                 return NULL;
  91
  92         // Read a couple of bytes
  93         size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
  94
  95         // Reset position
  96         r = fsetpos(f, &pos);
  97         if (r < 0)
  98                 return NULL;
  99
 100         // Check if we could read anything
 101         if (!bytes_read || bytes_read < sizeof(buffer))
 102                 return f;
 103
 104         // Analyze magic
 105         for (const struct compressor* c = compressors; c->open; c++) {
 106                 // Check if we have read enough data
 107                 if (bytes_read < c->magic_length)
 108                         continue;
 109
 110                 // Compare the magic value
 111                 r = memcmp(c->magic, buffer, c->magic_length);
 112                 if (r)
 113                         continue;
 114
 115                 // We found a match!
 116                 return c->open(f, mode);
 117         }
 118
 119         // Nothing seems to match
 120         errno = ENOTSUP;
 121         return f;
 122 }
 123
 124 struct xz_cookie {
 125         FILE* f;
 126         char mode;
 127         lzma_stream stream;
 128         int done;
 129
 130         uint8_t buffer[BUFFER_SIZE];
 131 };
 132
 133 static ssize_t xz_read(void* data, char* buffer, size_t size) {
 134         struct xz_cookie* cookie = (struct xz_cookie*)data;
 135         if (!cookie)
 136                 return -1;
 137
 138         // Do not read when mode is "w"
 139         if (cookie->mode == 'w')
 140                 return -1;
 141
 142         lzma_action action = LZMA_RUN;
 143
 144         // Set output to allocated buffer
 145         cookie->stream.next_out  = (uint8_t *)buffer;
 146         cookie->stream.avail_out = size;
 147
 148         while (1) {
 149                 // Read something when the input buffer is empty
 150                 if (cookie->stream.avail_in == 0) {
 151                         cookie->stream.next_in  = cookie->buffer;
 152                         cookie->stream.avail_in = fread(cookie->buffer,
 153                                 1, sizeof(cookie->buffer), cookie->f);
 154
 155                         // Break if the input file could not be read
 156                         if (ferror(cookie->f))
 157                                 return -1;
 158
 159                         // Finish after we have reached the end of the input file
 160                         if (feof(cookie->f))
 161                                 cookie->done = 1;
 162                 }
 163
 164                 lzma_ret ret = lzma_code(&cookie->stream, action);
 165
 166                 // If the stream has ended, we just send the
 167                 // remaining output and mark that we are done.
 168                 if (ret == LZMA_STREAM_END) {
 169                         cookie->done = 1;
 170                         return size - cookie->stream.avail_out;
 171                 }
 172
 173                 // Break on all other unexpected errors
 174                 if (ret != LZMA_OK)
 175                         return -1;
 176
 177                 // When we have read enough to fill the entire output buffer, we return
 178                 if (cookie->stream.avail_out == 0)
 179                         return size;
 180
 181                 if (cookie->done)
 182                         return -1;
 183         }
 184 }
 185
 186 static ssize_t xz_write(void* data, const char* buffer, size_t size) {
 187         struct xz_cookie* cookie = (struct xz_cookie*)data;
 188         if (!cookie)
 189                 return -1;
 190
 191         // Do not write when mode is "r"
 192         if (cookie->mode == 'r')
 193                 return -1;
 194
 195         // Return nothing when there is no input
 196         if (size == 0)
 197                 return 0;
 198
 199         // Set input to allocated buffer
 200         cookie->stream.next_in  = (uint8_t *)buffer;
 201         cookie->stream.avail_in = size;
 202
 203         while (1) {
 204                 cookie->stream.next_out  = cookie->buffer;
 205                 cookie->stream.avail_out = sizeof(cookie->buffer);
 206
 207                 lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
 208                 if (ret != LZMA_OK)
 209                         return -1;
 210
 211                 size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 212                 if (bytes_to_write) {
 213                         size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 214
 215                         if (bytes_written != bytes_to_write)
 216                                 return -1;
 217                 }
 218
 219                 // Report that all data has been written
 220                 if (cookie->stream.avail_in == 0)
 221                         return size;
 222         }
 223 }
 224
 225 static int xz_close(void* data) {
 226         struct xz_cookie* cookie = (struct xz_cookie*)data;
 227         if (!cookie)
 228                 return -1;
 229
 230         if (cookie->mode == 'w') {
 231                 while (1) {
 232                         cookie->stream.next_out  = cookie->buffer;
 233                         cookie->stream.avail_out = sizeof(cookie->buffer);
 234
 235                         lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
 236                         if (ret != LZMA_OK && ret != LZMA_STREAM_END)
 237                                 return -1;
 238
 239                         size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
 240                         if (bytes_to_write) {
 241                                 size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
 242
 243                                 if (bytes_written != bytes_to_write)
 244                                         return -1;
 245                         }
 246
 247                         if (ret == LZMA_STREAM_END)
 248                                 break;
 249                 }
 250         }
 251
 252         lzma_end(&cookie->stream);
 253
 254         // Close input file
 255         int r = fclose(cookie->f);
 256         free(cookie);
 257
 258         return r;
 259 }
 260
 261 static cookie_io_functions_t xz_functions = {
 262         .read  = xz_read,
 263         .write = xz_write,
 264         .seek  = NULL,
 265         .close = xz_close,
 266 };
 267
 268 FILE* pakfire_xzfopen(FILE* f, const char* mode) {
 269         lzma_ret ret;
 270
 271         if (!f) {
 272                 errno = EBADFD;
 273                 return NULL;
 274         }
 275
 276         if (!mode) {
 277                 errno = EINVAL;
 278                 return NULL;
 279         }
 280
 281         struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
 282         if (!cookie)
 283                 return NULL;
 284
 285         cookie->f = f;
 286         cookie->mode = *mode;
 287
 288         switch (cookie->mode) {
 289                 case 'r':
 290                         ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
 291                         break;
 292
 293                 case 'w':
 294                         ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
 295                         break;
 296
 297                 default:
 298                         errno = ENOTSUP;
 299                         goto ERROR;
 300         }
 301
 302         if (ret != LZMA_OK)
 303                 goto ERROR;
 304
 305         return fopencookie(cookie, mode, xz_functions);
 306
 307 ERROR:
 308         free(cookie);
 309         return NULL;
 310 }
 311
 312 // ZSTD
 313
 314 struct zstd_cookie {
 315         FILE* f;
 316         char mode;
 317         int done;
 318
 319         // Encoder
 320         ZSTD_CStream* cstream;
 321         ZSTD_inBuffer in;
 322
 323         // Decoder
 324         ZSTD_DStream* dstream;
 325         ZSTD_outBuffer out;
 326
 327         uint8_t buffer[BUFFER_SIZE];
 328 };
 329
 330 static ssize_t zstd_read(void* data, char* buffer, size_t size) {
 331         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 332         if (!cookie)
 333                 return -1;
 334
 335         // Do not read when mode is "w"
 336         if (cookie->mode == 'w')
 337                 return -1;
 338
 339         if (cookie->done)
 340                 return 0;
 341
 342         size_t r = 0;
 343
 344         // Configure output buffer
 345         cookie->out.dst = buffer;
 346         cookie->out.pos = 0;
 347         cookie->out.size = size;
 348
 349         while (1) {
 350                 if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
 351                         cookie->in.pos = 0;
 352                         cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
 353                 }
 354
 355                 if (r || cookie->in.size)
 356                         r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
 357
 358                 if (r == 0 && feof(cookie->f)) {
 359                         cookie->done = 1;
 360                         return cookie->out.pos;
 361                 }
 362
 363                 if (ZSTD_isError(r))
 364                         return -1;
 365
 366                 // Buffer full
 367                 if (cookie->out.pos == size)
 368                         return size;
 369         }
 370 }
 371
 372 static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
 373         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 374         if (!cookie)
 375                 return -1;
 376
 377         // Do not write when mode is "r"
 378         if (cookie->mode == 'r')
 379                 return -1;
 380
 381         // Return nothing when there is no input
 382         if (size == 0)
 383                 return 0;
 384
 385         // Configure input buffer
 386         cookie->in.src = buffer;
 387         cookie->in.pos = 0;
 388         cookie->in.size = size;
 389
 390         while (1) {
 391                 cookie->out.pos = 0;
 392
 393                 size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
 394                 if (ZSTD_isError(r))
 395                         return -1;
 396
 397                 if (cookie->out.pos > 0) {
 398                         size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 399
 400                         if (bytes_written != cookie->out.pos)
 401                                 return -1;
 402                 }
 403
 404                 // Return when all input has been written
 405                 if (cookie->in.pos == size)
 406                         return size;
 407         }
 408 }
 409
 410 static int zstd_close(void* data) {
 411         struct zstd_cookie* cookie = (struct zstd_cookie*)data;
 412         if (!cookie)
 413                 return -1;
 414
 415         if (cookie->mode == 'w') {
 416                 while (1) {
 417                         // Reset output buffer
 418                         cookie->out.pos = 0;
 419
 420                         size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
 421                         if (ZSTD_isError(r))
 422                                 return -1;
 423
 424                         if (cookie->out.pos > 0) {
 425                                 size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
 426
 427                                 if (bytes_written != cookie->out.pos)
 428                                         return -1;
 429                         }
 430
 431                         if (r == 0)
 432                                 break;
 433                 }
 434         }
 435
 436         int r = fclose(cookie->f);
 437
 438         // Free everything
 439         if (cookie->cstream)
 440                 ZSTD_freeCStream(cookie->cstream);
 441         if (cookie->dstream)
 442                 ZSTD_freeDStream(cookie->dstream);
 443         free(cookie);
 444
 445         return r;
 446 }
 447
 448 static cookie_io_functions_t zstd_functions = {
 449         .read  = zstd_read,
 450         .write = zstd_write,
 451         .seek  = NULL,
 452         .close = zstd_close,
 453 };
 454
 455 FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
 456         if (!f) {
 457                 errno = EBADFD;
 458                 return NULL;
 459         }
 460
 461         if (!mode) {
 462                 errno = EINVAL;
 463                 return NULL;
 464         }
 465
 466         struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
 467         if (!cookie)
 468                 return NULL;
 469
 470         cookie->f = f;
 471         cookie->mode = *mode;
 472
 473         size_t r;
 474         switch (cookie->mode) {
 475                 case 'r':
 476                         // Allocate stream
 477                         cookie->dstream = ZSTD_createDStream();
 478                         if (!cookie->dstream)
 479                                 goto ERROR;
 480
 481                         // Initialize stream
 482                         r = ZSTD_initDStream(cookie->dstream);
 483                         if (ZSTD_isError(r))
 484                                 goto ERROR;
 485
 486                         cookie->in.src = cookie->buffer;
 487                         cookie->in.pos = 0;
 488                         cookie->in.size = 0;
 489                         break;
 490
 491                 case 'w':
 492                         // Allocate stream
 493                         cookie->cstream = ZSTD_createCStream();
 494                         if (!cookie->cstream)
 495                                 goto ERROR;
 496
 497                         // Initialize stream
 498                         r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
 499                         if (ZSTD_isError(r))
 500                                 goto ERROR;
 501
 502                         cookie->out.dst = cookie->buffer;
 503                         cookie->out.pos = 0;
 504                         cookie->out.size = sizeof(cookie->buffer);
 505                         break;
 506
 507                 default:
 508                         errno = ENOTSUP;
 509                         goto ERROR;
 510         }
 511
 512         return fopencookie(cookie, mode, zstd_functions);
 513
 514 ERROR:
 515         if (cookie->cstream)
 516                 ZSTD_freeCStream(cookie->cstream);
 517         if (cookie->dstream)
 518                 ZSTD_freeDStream(cookie->dstream);
 519         free(cookie);
 520
 521         return NULL;
 522 }
 523
 524 /*
 525         Helper function to conditionally walk through an archive
 526         and perform actions based on the callback.
 527 */
 528 int pakfire_walk(struct pakfire_ctx* ctx, struct archive* archive,
 529                 pakfire_walk_callback callback, pakfire_walk_filter_callback filter_callback,
 530                 void* p) {
 531         struct archive_entry* entry = NULL;
 532         const char* path = NULL;
 533         int r;
 534
 535         // Walk through the archive
 536         for (;;) {
 537                 r = archive_read_next_header(archive, &entry);
 538
 539                 // Handle the return code
 540                 switch (r) {
 541                         // Fall through if everything is okay
 542                         case ARCHIVE_OK:
 543                                 break;
 544
 545                         // Return OK when we reached the end of the archive
 546                         case ARCHIVE_EOF:
 547                                 return 0;
 548
 549                         // Raise any other errors
 550                         default:
 551                                 return r;
 552                 }
 553
 554                 path = archive_entry_pathname(entry);
 555
 556                 CTX_DEBUG(ctx, "Walking through %s...\n", path);
 557
 558                 // Call the filter callback before we call the actual callback
 559                 if (filter_callback) {
 560                         r = filter_callback(ctx, archive, entry, p);
 561
 562                         // Handle the return code
 563                         switch (r) {
 564                                 case PAKFIRE_WALK_OK:
 565                                         break;
 566
 567                                 case PAKFIRE_WALK_END:
 568                                         CTX_DEBUG(ctx, "Filter callback sent END\n");
 569                                         return 0;
 570
 571                                 case PAKFIRE_WALK_SKIP:
 572                                         CTX_DEBUG(ctx, "Filter callback sent SKIP\n");
 573                                         continue;
 574
 575                                 case PAKFIRE_WALK_DONE:
 576                                         CTX_DEBUG(ctx, "Filter callback sent DONE\n");
 577
 578                                         // Clear the callback function
 579                                         filter_callback = NULL;
 580                                         break;
 581
 582                                 case PAKFIRE_WALK_AGAIN:
 583                                         CTX_DEBUG(ctx, "Filter callback sent AGAIN\n");
 584                                         return -EAGAIN;
 585
 586                                 // Raise any other errors
 587                                 default:
 588                                         CTX_DEBUG(ctx, "Filter callback returned an error: %d\n", r);
 589                                         return r;
 590                         }
 591                 }
 592
 593                 // Run callback
 594                 if (callback) {
 595                         r = callback(ctx, archive, entry, p);
 596
 597                         // Handle the return code
 598                         switch (r) {
 599                                 case PAKFIRE_WALK_OK:
 600                                         break;
 601
 602                                 case PAKFIRE_WALK_DONE:
 603                                         CTX_DEBUG(ctx, "Callback sent DONE\n");
 604                                         return 0;
 605
 606                                 // Raise any other errors
 607                                 default:
 608                                         return r;
 609                         }
 610                 }
 611         }
 612
 613         return 0;
 614 }
 615
 616 // Common extraction
 617
 618 struct pakfire_extract {
 619         // Reference to Pakfire
 620         struct pakfire* pakfire;
 621
 622         // Flags
 623         int flags;
 624
 625         // The archive to extract
 626         struct archive* archive;
 627
 628         // The filelist of all extracted files
 629         struct pakfire_filelist* filelist;
 630
 631         // Prepend this prefix
 632         const char* prefix;
 633
 634         // The writer
 635         struct archive* writer;
 636
 637         // The progress indicator
 638         struct pakfire_progress* progress;
 639 };
 640
 641 static void pakfire_extract_progress(void* p) {
 642         struct pakfire_extract* data = (struct pakfire_extract*)p;
 643
 644         // Fetch how many bytes have been read
 645         const size_t position = archive_filter_bytes(data->archive, -1);
 646
 647         // Update progress
 648         pakfire_progress_update(data->progress, position);
 649 }
 650
 651 static int __pakfire_extract(struct pakfire_ctx* ctx, struct archive* a,
 652                 struct archive_entry* entry, void* p) {
 653         struct pakfire_file* file = NULL;
 654         struct vfs_cap_data cap_data = {};
 655         char buffer[PATH_MAX];
 656         int r;
 657
 658         struct pakfire_extract* data = (struct pakfire_extract*)p;
 659
 660         // Fetch path
 661         const char* path = archive_entry_pathname(entry);
 662
 663         // Make sure we have a leading slash on the filelist
 664         if (!pakfire_string_startswith(path, "/")) {
 665                 r = pakfire_string_format(buffer, "/%s", path);
 666                 if (r)
 667                         goto ERROR;
 668
 669                 // Store the new name
 670                 archive_entry_set_pathname(entry, buffer);
 671
 672                 // Update the path pointer
 673                 path = archive_entry_pathname(entry);
 674         }
 675
 676         // Generate a file object
 677         r = pakfire_file_create_from_archive_entry(&file, data->pakfire, entry);
 678         if (r)
 679                 goto ERROR;
 680
 681         // Add entry to filelist (if requested)
 682         if (data->filelist) {
 683                 // Append the file to the list
 684                 r = pakfire_filelist_add(data->filelist, file);
 685                 if (r)
 686                         goto ERROR;
 687         }
 688
 689         const int configfile = pakfire_file_has_flag(file, PAKFIRE_FILE_CONFIG);
 690
 691         // Prepend the prefix
 692         if (*data->prefix) {
 693                 // Compose file path
 694                 r = pakfire_path_append(buffer, data->prefix, path);
 695                 if (r) {
 696                         CTX_ERROR(ctx, "Could not compose file path: %m\n");
 697                         goto ERROR;
 698                 }
 699
 700                 // Set file path
 701                 archive_entry_set_pathname(entry, buffer);
 702
 703                 // Update hardlink destination
 704                 const char* link = archive_entry_hardlink(entry);
 705                 if (link) {
 706                         r = pakfire_path_append(buffer, data->prefix, link);
 707                         if (r) {
 708                                 CTX_ERROR(ctx, "Could not compose hardlink path: %m\n");
 709                                 goto ERROR;
 710                         }
 711
 712                         // Set hardlink path
 713                         archive_entry_set_hardlink(entry, buffer);
 714                 }
 715         }
 716
 717         if (configfile) {
 718                 // Fetch path again since we changed it
 719                 path = archive_entry_pathname(entry);
 720
 721                 if (pakfire_path_exists(path)) {
 722                         CTX_DEBUG(ctx, "The configuration file %s exists\n",
 723                                 pakfire_file_get_path(file));
 724
 725                         r = pakfire_string_format(buffer, "%s.paknew", path);
 726                         if (r) {
 727                                 CTX_ERROR(ctx, "Could not compose path for configuration file: %m\n");
 728                                 goto ERROR;
 729                         }
 730
 731                         // Set the path again
 732                         archive_entry_set_pathname(entry, buffer);
 733                 }
 734         }
 735
 736         // Create file & extract payload
 737         if (data->writer) {
 738                 // Fetch path again since we changed it
 739                 path = archive_entry_pathname(entry);
 740
 741                 CTX_DEBUG(ctx, "Extracting %s\n", path);
 742
 743                 // Remove any extended attributes which we never write to disk
 744                 archive_entry_xattr_clear(entry);
 745
 746                 // Set capabilities
 747                 if (pakfire_file_has_caps(file)) {
 748                         r = pakfire_file_write_fcaps(file, &cap_data);
 749                         if (r)
 750                                 goto ERROR;
 751
 752                         // Store capabilities in archive entry
 753                         archive_entry_xattr_add_entry(entry, "security.capability",
 754                                 &cap_data, sizeof(cap_data));
 755                 }
 756
 757                 // Write payload
 758                 r = archive_read_extract2(data->archive, entry, data->writer);
 759                 switch (r) {
 760                         case ARCHIVE_OK:
 761                                 r = 0;
 762                                 break;
 763
 764                         case ARCHIVE_WARN:
 765                                 CTX_ERROR(ctx, "%s\n", archive_error_string(data->writer));
 766
 767                                 // Pretend everything has been okay
 768                                 r = 0;
 769                                 break;
 770
 771                         case ARCHIVE_FATAL:
 772                                 CTX_ERROR(ctx, "%s\n", archive_error_string(data->writer));
 773                                 r = 1;
 774                                 break;
 775                 }
 776         }
 777
 778 ERROR:
 779         if (file)
 780                 pakfire_file_unref(file);
 781
 782         return r;
 783 }
 784
 785 int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
 786                 size_t size, struct pakfire_filelist* filelist,
 787                 const char* prefix, const char* message,
 788                 pakfire_walk_filter_callback filter_callback, int flags) {
 789         int progress_flags = PAKFIRE_PROGRESS_SHOW_PERCENTAGE;
 790         int r = 1;
 791
 792         struct pakfire_ctx* ctx = pakfire_ctx(pakfire);
 793
 794         // Use / if no prefix is set
 795         if (!prefix)
 796                 prefix = "/";
 797
 798         struct pakfire_extract data = {
 799                 .pakfire  = pakfire,
 800                 .archive  = archive,
 801                 .filelist = filelist,
 802                 .prefix   = prefix,
 803                 .flags    = flags,
 804                 .writer   = NULL,
 805         };
 806
 807         // Is this a dry run?
 808         const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;
 809
 810         // Allocate writer
 811         if (!dry_run) {
 812                 data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
 813                 if (!data.writer) {
 814                         ERROR(pakfire, "Could not create disk writer: %m\n");
 815                         r = 1;
 816                         goto ERROR;
 817                 }
 818         }
 819
 820         // Should we show any progress?
 821         if (flags & PAKFIRE_EXTRACT_NO_PROGRESS)
 822                 progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
 823
 824         // Show throughput?
 825         if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT)
 826                 progress_flags |= PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED;
 827
 828         // Create the progress indicator
 829         r = pakfire_progress_create(&data.progress, ctx, progress_flags, NULL);
 830         if (r)
 831                 goto ERROR;
 832
 833         // Set the title
 834         r = pakfire_progress_set_title(data.progress, "%s", message);
 835         if (r)
 836                 goto ERROR;
 837
 838         // Register progress callback
 839         archive_read_extract_set_progress_callback(data.archive,
 840                 pakfire_extract_progress, &data);
 841
 842         // Start progress
 843         r = pakfire_progress_start(data.progress, size);
 844         if (r)
 845                 goto ERROR;
 846
 847         // Walk through the entire archive
 848         r = pakfire_walk(ctx, archive, __pakfire_extract, filter_callback, &data);
 849         if (r)
 850                 goto ERROR;
 851
 852         // Finish the progress
 853         r = pakfire_progress_finish(data.progress);
 854         if (r)
 855                 goto ERROR;
 856
 857 ERROR:
 858         if (data.progress)
 859                 pakfire_progress_unref(data.progress);
 860         if (data.writer)
 861                 archive_write_free(data.writer);
 862         if (ctx)
 863                 pakfire_ctx_unref(ctx);
 864
 865         return r;
 866 }
 867
 868 // Common compression
 869
 870 struct pakfire_compress {
 871         // Reference to Pakfire
 872         struct pakfire* pakfire;
 873
 874         // Flags
 875         int flags;
 876
 877         // The archive to write to
 878         struct archive* archive;
 879
 880         // Resolver for hardlinks
 881         struct archive_entry_linkresolver* linkresolver;
 882
 883         // The filelist of all files to write
 884         struct pakfire_filelist* filelist;
 885
 886         // The progress indicator
 887         struct pakfire_progress* progress;
 888
 889         // Digests to write to the archive
 890         int digests;
 891 };
 892
 893 static int pakfire_copy_data_from_file(struct pakfire* pakfire,
 894                 struct archive* archive, FILE* f) {
 895         char buffer[BUFFER_SIZE];
 896
 897         ssize_t bytes_read = 0;
 898         ssize_t bytes_written = 0;
 899
 900         // Read file from the very beginning - also allows calling this multiple times
 901         rewind(f);
 902
 903         // Loop through the entire length of the file
 904         while (!feof(f)) {
 905                 // Read a block from file
 906                 bytes_read = fread(buffer, 1, sizeof(buffer), f);
 907
 908                 // Check if any error occured
 909                 if (ferror(f)) {
 910                         ERROR(pakfire, "Read error: %m\n");
 911                         return -errno;
 912                 }
 913
 914                 // Write the block to the archive
 915                 bytes_written = archive_write_data(archive, buffer, bytes_read);
 916                 if (bytes_written < bytes_read) {
 917                         ERROR(pakfire, "Write error: %s\n", archive_error_string(archive));
 918                         return -errno;
 919                 }
 920         }
 921
 922         return 0;
 923 }
 924
 925 static int __pakfire_compress_entry(struct pakfire* pakfire, struct pakfire_file* file,
 926                 struct pakfire_compress* data, struct archive_entry* entry) {
 927         FILE* f = NULL;
 928         int r;
 929
 930         const char* path = archive_entry_pathname(entry);
 931
 932         // Remove any leading slahes
 933         while (*path == '/')
 934                 path++;
 935
 936         archive_entry_set_pathname(entry, path);
 937
 938         // Write the header
 939         r = archive_write_header(data->archive, entry);
 940         if (r) {
 941                 ERROR(pakfire, "Error writing file header: %s\n",
 942                         archive_error_string(data->archive));
 943                 goto ERROR;
 944         }
 945
 946         // Copy the data if there is any
 947         if (archive_entry_size(entry)) {
 948                 // Open the file
 949                 f = pakfire_file_open(file);
 950                 if (!f) {
 951                         r = 1;
 952                         goto ERROR;
 953                 }
 954
 955                 // Copy the payload into the archive
 956                 r = pakfire_copy_data_from_file(pakfire, data->archive, f);
 957                 if (r)
 958                         goto ERROR;
 959         }
 960
 961         // Write trailer
 962         r = archive_write_finish_entry(data->archive);
 963         if (r)
 964                 goto ERROR;
 965
 966 ERROR:
 967         if (f)
 968                 fclose(f);
 969
 970         return r;
 971 }
 972
 973 static int __pakfire_compress(struct pakfire* pakfire, struct pakfire_file* file, void* p) {
 974         struct archive_entry* entry = NULL;
 975         struct archive_entry* sparse_entry = NULL;
 976         int r = 1;
 977
 978         struct pakfire_compress* data = (struct pakfire_compress*)p;
 979
 980         // Fetch the file size
 981         const size_t size = pakfire_file_get_size(file);
 982
 983         // Generate file metadata into an archive entry
 984         entry = pakfire_file_archive_entry(file, data->digests);
 985         if (!entry) {
 986                 r = 1;
 987                 goto ERROR;
 988         }
 989
 990         // Perform search for hardlinks
 991         archive_entry_linkify(data->linkresolver, &entry, &sparse_entry);
 992
 993         // Write the main entry
 994         if (entry) {
 995                 r = __pakfire_compress_entry(pakfire, file, data, entry);
 996                 if (r)
 997                         goto ERROR;
 998         }
 999
1000         // Write the sparse entry
1001         if (sparse_entry) {
1002                 r = __pakfire_compress_entry(pakfire, file, data, sparse_entry);
1003                 if (r)
1004                         goto ERROR;
1005         }
1006
1007         // Query the link resolver for any more entries
1008         for (;;) {
1009                 // Free the entry
1010                 if (entry) {
1011                         archive_entry_free(entry);
1012                         entry = NULL;
1013                 }
1014
1015                 // Free the sparse entry
1016                 if (sparse_entry) {
1017                         archive_entry_free(sparse_entry);
1018                         sparse_entry = NULL;
1019                 }
1020
1021                 // Fetch the next entry
1022                 archive_entry_linkify(data->linkresolver, &entry, &sparse_entry);
1023                 if (!entry)
1024                         break;
1025
1026                 // Write the entry to the archive
1027                 r = __pakfire_compress_entry(pakfire, file, data, entry);
1028                 if (r)
1029                         goto ERROR;
1030         }
1031
1032         // Update the progress
1033         pakfire_progress_increment(data->progress, size);
1034
1035 ERROR:
1036         if (entry)
1037                 archive_entry_free(entry);
1038         if (sparse_entry)
1039                 archive_entry_free(sparse_entry);
1040
1041         return r;
1042 }
1043
1044 int pakfire_compress(struct pakfire* pakfire, struct archive* archive,
1045                 struct pakfire_filelist* filelist, const char* message, int flags, int digests) {
1046         int progress_flags =
1047                 PAKFIRE_PROGRESS_SHOW_PERCENTAGE |
1048                 PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED;
1049         int r = 1;
1050
1051         struct pakfire_ctx* ctx = pakfire_ctx(pakfire);
1052
1053         struct pakfire_compress data = {
1054                 .pakfire  = pakfire,
1055                 .archive  = archive,
1056                 .filelist = filelist,
1057                 .flags    = flags,
1058                 .digests  = digests,
1059         };
1060
1061         // Should we show a progress bar?
1062         if (flags & PAKFIRE_COMPRESS_NO_PROGRESS)
1063                 progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
1064
1065         // Should we show throughput?
1066         if (flags & PAKFIRE_COMPRESS_SHOW_THROUGHPUT)
1067                 progress_flags |= PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED;
1068
1069         // Fetch the length of the filelist
1070         const size_t size = pakfire_filelist_total_size(filelist);
1071
1072         // Create the progress indicator
1073         r = pakfire_progress_create(&data.progress, ctx, progress_flags, NULL);
1074         if (r)
1075                 goto ERROR;
1076
1077         // Set progress title
1078         r = pakfire_progress_set_title(data.progress, "%s", message);
1079         if (r)
1080                 goto ERROR;
1081
1082         // Start progress
1083         r = pakfire_progress_start(data.progress, size);
1084         if (r)
1085                 goto ERROR;
1086
1087         // Setup the link resolver
1088         data.linkresolver = archive_entry_linkresolver_new();
1089         if (!data.linkresolver) {
1090                 ERROR(pakfire, "Could not setup link resolver: m\n");
1091                 goto ERROR;
1092         }
1093
1094         // Set the link resolver strategy
1095         archive_entry_linkresolver_set_strategy(data.linkresolver, archive_format(archive));
1096
1097         // Walk through the entire filelist
1098         r = pakfire_filelist_walk(filelist, __pakfire_compress, &data, 0);
1099         if (r)
1100                 goto ERROR;
1101
1102         // Finish the progress
1103         if (data.progress)
1104                 pakfire_progress_finish(data.progress);
1105
1106 ERROR:
1107         if (data.progress)
1108                 pakfire_progress_unref(data.progress);
1109         if (data.linkresolver)
1110                 archive_entry_linkresolver_free(data.linkresolver);
1111         if (ctx)
1112                 pakfire_ctx_unref(ctx);
1113
1114         return r;
1115 }
1116
1117 int pakfire_compress_create_archive(struct pakfire* pakfire, struct archive** archive,
1118                 FILE* f, const enum pakfire_compressions compression, const unsigned int level) {
1119         struct archive* a = NULL;
1120         char value[16];
1121         int r;
1122
1123         // Open a new archive
1124         a = archive_write_new();
1125         if (!a) {
1126                 ERROR(pakfire, "archive_write_new() failed\n");
1127                 r = 1;
1128                 goto ERROR;
1129         }
1130
1131         // Use the PAX format
1132         r = archive_write_set_format_pax(a);
1133         if (r) {
1134                 ERROR(pakfire, "Could not set format to PAX: %s\n", archive_error_string(a));
1135                 goto ERROR;
1136         }
1137
1138         // Store any extended attributes in the SCHILY headers
1139         r = archive_write_set_format_option(a, "pax", "xattrheader", "SCHILY");
1140         if (r) {
1141                 ERROR(pakfire, "Could not set xattrheader option: %s\n", archive_error_string(a));
1142                 goto ERROR;
1143         }
1144
1145         switch (compression) {
1146                 case PAKFIRE_COMPRESS_ZSTD:
1147                         // Enable Zstd
1148                         r = archive_write_add_filter_zstd(a);
1149                         if (r) {
1150                                 ERROR(pakfire, "Could not enable Zstandard compression: %s\n",
1151                                         archive_error_string(a));
1152                                 goto ERROR;
1153                         }
1154
1155                         // Do not pad the last block
1156                         archive_write_set_bytes_in_last_block(a, 1);
1157
1158                         // Set compression level
1159                         if (level) {
1160                                 r = pakfire_string_format(value, "%u", level);
1161                                 if (r)
1162                                         goto ERROR;
1163
1164                                 r = archive_write_set_filter_option(a, NULL, "compression-level", value);
1165                                 if (r) {
1166                                         ERROR(pakfire, "Could not set Zstandard compression level: %s\n",
1167                                                 archive_error_string(a));
1168                                         goto ERROR;
1169                                 }
1170                         }
1171
1172 #if ARCHIVE_VERSION_NUMBER >= 3006000
1173                         // Fetch numbers of processors
1174                         long processors = sysconf(_SC_NPROCESSORS_ONLN);
1175
1176                         if (processors > 1) {
1177                                 r = pakfire_string_format(value, "%ld", processors);
1178                                 if (r) {
1179                                         ERROR(pakfire, "Could not format threads: %m\n");
1180                                         goto ERROR;
1181                                 }
1182
1183                                 // Try using multiple threads
1184                                 r = archive_write_set_filter_option(a, NULL, "threads", value);
1185                                 if (r) {
1186                                         ERROR(pakfire, "Could not enable %ld threads for compression: %s\n",
1187                                                 processors, archive_error_string(a));
1188                                         goto ERROR;
1189                                 }
1190                         }
1191 #endif
1192         }
1193
1194         // Write archive to f
1195         if (f) {
1196                 r = archive_write_open_FILE(a, f);
1197                 if (r) {
1198                         ERROR(pakfire, "archive_write_open_FILE() failed: %s\n", archive_error_string(a));
1199                         goto ERROR;
1200                 }
1201         }
1202
1203         // Success
1204         *archive = a;
1205
1206         return 0;
1207
1208 ERROR:
1209         if (a)
1210                 archive_write_free(a);
1211
1212         return r;
1213 }