src/libpakfire/xfer.c

   1 /*#############################################################################
   2 #                                                                             #
   3 # Pakfire - The IPFire package management system                              #
   4 # Copyright (C) 2023 Pakfire development team                                 #
   5 #                                                                             #
   6 # This program is free software: you can redistribute it and/or modify        #
   7 # it under the terms of the GNU General Public License as published by        #
   8 # the Free Software Foundation, either version 3 of the License, or           #
   9 # (at your option) any later version.                                         #
  10 #                                                                             #
  11 # This program is distributed in the hope that it will be useful,             #
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  14 # GNU General Public License for more details.                                #
  15 #                                                                             #
  16 # You should have received a copy of the GNU General Public License           #
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  18 #                                                                             #
  19 #############################################################################*/
  20
  21 #include <errno.h>
  22 #include <fcntl.h>
  23 #include <limits.h>
  24 #include <sys/queue.h>
  25 #include <utime.h>
  26
  27 #include <curl/curl.h>
  28
  29 #include <json.h>
  30
  31 #include <openssl/err.h>
  32 #include <openssl/evp.h>
  33
  34 #include <pakfire/ctx.h>
  35 #include <pakfire/mirrorlist.h>
  36 #include <pakfire/path.h>
  37 #include <pakfire/progress.h>
  38 #include <pakfire/string.h>
  39 #include <pakfire/util.h>
  40 #include <pakfire/xfer.h>
  41
  42 struct pakfire_xfer {
  43         struct pakfire_ctx* ctx;
  44         int nrefs;
  45
  46         // Reference to the HTTP client
  47         struct pakfire_httpclient* client;
  48
  49         // Reference to the progress indicator
  50         struct pakfire_progress* progress;
  51
  52         // cURL handle
  53         CURL* handle;
  54
  55         // Headers
  56         struct curl_slist* headers;
  57
  58         // URL
  59         CURLU* fullurl;
  60
  61         char url[PATH_MAX];
  62         char title[NAME_MAX];
  63         char path[PATH_MAX];
  64         pakfire_xfer_flags_t flags;
  65         int tries;
  66
  67         // POST MIME Object
  68         curl_mime* mime;
  69
  70         // Xfer direction
  71         enum {
  72                 PAKFIRE_XFER_DOWNLOAD = 0,
  73                 PAKFIRE_XFER_UPLOAD   = 1,
  74         } direction;
  75
  76         // Size
  77         size_t expected_size;
  78         size_t xferred;
  79
  80         // File handles for streams
  81         FILE* fin;
  82         FILE* fout;
  83
  84         // Crypto Stuff
  85         EVP_MD_CTX* evp;
  86         const EVP_MD* md;
  87         unsigned char computed_digest[EVP_MAX_MD_SIZE];
  88         unsigned int computed_digest_length;
  89         unsigned char expected_digest[EVP_MAX_MD_SIZE];
  90         unsigned int expected_digest_length;
  91
  92         // Mirrors
  93         char baseurl[PATH_MAX];
  94         struct pakfire_mirrorlist* mirrors;
  95         struct pakfire_mirror* mirror;
  96
  97         // Effective URL
  98         const char* effective_url;
  99
 100         // Authentication
 101         unsigned int auth;
 102 };
 103
 104 static void pakfire_xfer_free(struct pakfire_xfer* xfer) {
 105         // Close any streams
 106         if (xfer->fin)
 107                 fclose(xfer->fin);
 108
 109         // Free OpenSSL EVP context
 110         if (xfer->evp)
 111                 EVP_MD_CTX_free(xfer->evp);
 112
 113         // cURL stuff
 114         if (xfer->handle)
 115                 curl_easy_cleanup(xfer->handle);
 116         if (xfer->headers)
 117                 curl_slist_free_all(xfer->headers);
 118         if (xfer->mime)
 119                 curl_mime_free(xfer->mime);
 120         if (xfer->fullurl)
 121                 curl_url_cleanup(xfer->fullurl);
 122
 123         if (xfer->mirror)
 124                 pakfire_mirror_unref(xfer->mirror);
 125         if (xfer->mirrors)
 126                 pakfire_mirrorlist_unref(xfer->mirrors);
 127         if (xfer->progress)
 128                 pakfire_progress_unref(xfer->progress);
 129         if (xfer->client)
 130                 pakfire_httpclient_unref(xfer->client);
 131         if (xfer->ctx)
 132                 pakfire_ctx_unref(xfer->ctx);
 133
 134         free(xfer);
 135 }
 136
 137 #ifdef ENABLE_DEBUG
 138 static int pakfire_xfer_debug_callback(CURL *handle, curl_infotype type,
 139                 char* data, size_t size, void* private) {
 140         struct pakfire_ctx* ctx = private;
 141
 142         switch (type) {
 143                 case CURLINFO_TEXT:
 144                         CTX_DEBUG(ctx, "cURL: %.*s", (int)size, data);
 145                         break;
 146
 147                 // Log headers
 148                 case CURLINFO_HEADER_IN:
 149                         CTX_DEBUG(ctx, "cURL: < %.*s", (int)size, data);
 150                         break;
 151
 152                 case CURLINFO_HEADER_OUT:
 153                         CTX_DEBUG(ctx, "cURL: > %.*s", (int)size, data);
 154                         break;
 155
 156                 // Ignore everything else
 157                 default:
 158                         break;
 159         }
 160
 161         return 0;
 162 }
 163 #endif
 164
 165 static size_t pakfire_xfer_read(char* data, size_t size, size_t nmemb, void* p) {
 166         struct pakfire_xfer* xfer = p;
 167
 168         return fread(data, size, nmemb, xfer->fout);
 169 }
 170
 171 static int pakfire_xfer_seek(void* p, curl_off_t offset, int origin) {
 172         struct pakfire_xfer* xfer = p;
 173         int r;
 174
 175         // Perform the seek
 176         r = fseek(xfer->fout, (long)offset, origin);
 177         if (r < 0)
 178                 return CURL_SEEKFUNC_CANTSEEK;
 179
 180         return CURL_SEEKFUNC_OK;
 181 }
 182
 183 static size_t pakfire_xfer_write(
 184                 char* data, size_t size, size_t nmemb, void* p) {
 185         struct pakfire_xfer* xfer = p;
 186         struct pakfire_ctx* ctx = xfer->ctx;
 187         int r;
 188
 189         // Do not write empty blocks
 190         if (!nmemb)
 191                 return nmemb;
 192
 193         // Update message digest
 194         if (xfer->evp) {
 195                 r = EVP_DigestUpdate(xfer->evp, data, nmemb);
 196                 if (r != 1) {
 197                         CTX_ERROR(ctx, "EVP_DigestUpdate failed: %s\n",
 198                                 ERR_error_string(ERR_get_error(), NULL));
 199                         return 0;
 200                 }
 201         }
 202
 203         // If there is no output steam, we just pretent that we have consumed the data
 204         if (!xfer->fin)
 205                 return nmemb;
 206
 207         // Write everything to the allocated file descriptor
 208         return fwrite(data, size, nmemb, xfer->fin);
 209 }
 210
 211 static int pakfire_xfer_setup(struct pakfire_xfer* xfer) {
 212         struct pakfire_config* config = NULL;
 213         const char* proxy = NULL;
 214         int r;
 215
 216         CURLSH* share = pakfire_httpclient_share(xfer->client);
 217
 218         // Configure the share handle
 219         r = curl_easy_setopt(xfer->handle, CURLOPT_SHARE, share);
 220         if (r) {
 221                 CTX_ERROR(xfer->ctx, "Could not configure cURL share handle: %s\n",
 222                         curl_easy_strerror(r));
 223                 return r;
 224         }
 225
 226         // Fetch global configuration
 227         config = pakfire_ctx_get_config(xfer->ctx);
 228
 229         // Set global configuration
 230         if (config) {
 231                 proxy = pakfire_config_get(config, "general", "proxy", NULL);
 232                 if (proxy)
 233                         curl_easy_setopt(xfer->handle, CURLOPT_PROXY, proxy);
 234         }
 235
 236         // Be a good net citizen and set a user agent
 237         curl_easy_setopt(xfer->handle, CURLOPT_USERAGENT, PACKAGE_NAME "/" PACKAGE_VERSION);
 238
 239 #ifdef ENABLE_DEBUG
 240         // Enable logging/debugging
 241         curl_easy_setopt(xfer->handle, CURLOPT_VERBOSE, 1L);
 242
 243         curl_easy_setopt(xfer->handle, CURLOPT_DEBUGFUNCTION, pakfire_xfer_debug_callback);
 244         curl_easy_setopt(xfer->handle, CURLOPT_DEBUGDATA, xfer->ctx);
 245 #endif
 246
 247         // Limit protocols to HTTPS, HTTP, FTP and FILE
 248         curl_easy_setopt(xfer->handle, CURLOPT_PROTOCOLS_STR, "HTTPS,HTTP,FTP,FILE");
 249
 250         // Allow all support encodings
 251         curl_easy_setopt(xfer->handle, CURLOPT_ACCEPT_ENCODING, "");
 252
 253         // Reference back to this xfer
 254         curl_easy_setopt(xfer->handle, CURLOPT_PRIVATE, xfer);
 255
 256         // Follow any redirects
 257         curl_easy_setopt(xfer->handle, CURLOPT_FOLLOWLOCATION, 1);
 258
 259         // Only follow up to 30 redirects
 260         curl_easy_setopt(xfer->handle, CURLOPT_MAXREDIRS, 30L);
 261
 262         // Read any data from a callback function
 263         curl_easy_setopt(xfer->handle,
 264                 CURLOPT_READFUNCTION, pakfire_xfer_read);
 265         curl_easy_setopt(xfer->handle, CURLOPT_READDATA, xfer);
 266
 267         // Write all data to the callback function
 268         curl_easy_setopt(xfer->handle,
 269                 CURLOPT_WRITEFUNCTION, pakfire_xfer_write);
 270         curl_easy_setopt(xfer->handle, CURLOPT_WRITEDATA, xfer);
 271
 272         // Register the seek callback
 273         curl_easy_setopt(xfer->handle,
 274                 CURLOPT_SEEKFUNCTION, pakfire_xfer_seek);
 275         curl_easy_setopt(xfer->handle, CURLOPT_SEEKDATA, xfer);
 276
 277         // Success
 278         r = 0;
 279
 280         // Cleanup
 281         if (config)
 282                 pakfire_config_unref(config);
 283
 284         return r;
 285 }
 286
 287 int pakfire_xfer_create(struct pakfire_xfer** xfer, struct pakfire_ctx* ctx,
 288                         struct pakfire_httpclient* client, const char* url) {
 289         struct pakfire_xfer* x = NULL;
 290         int r;
 291
 292         // Allocate a new xfer
 293         x = calloc(1, sizeof(*x));
 294         if (!x)
 295                 return -errno;
 296
 297         // Store a reference to the context
 298         x->ctx = pakfire_ctx_ref(ctx);
 299
 300         // Initialize the reference counter
 301         x->nrefs = 1;
 302
 303         // Store a reference to the HTTP client
 304         x->client = pakfire_httpclient_ref(client);
 305
 306         // Store the URL
 307         r = pakfire_string_set(x->url, url);
 308         if (r)
 309                 goto ERROR;
 310
 311         // Allocate a handle
 312         x->handle = curl_easy_init();
 313         if (!x->handle) {
 314                 r = 1;
 315                 goto ERROR;
 316         }
 317
 318         // Allocate the full URL
 319         x->fullurl = curl_url();
 320         if (!x->fullurl) {
 321                 r = -errno;
 322                 goto ERROR;
 323         }
 324
 325         // Setup the xfer
 326         r = pakfire_xfer_setup(x);
 327         if (r)
 328                 goto ERROR;
 329
 330         // Return the reference
 331         *xfer = x;
 332
 333         return 0;
 334
 335 ERROR:
 336         if (x)
 337                 pakfire_xfer_unref(x);
 338
 339         return r;
 340 }
 341
 342 struct pakfire_xfer* pakfire_xfer_ref(struct pakfire_xfer* xfer) {
 343         ++xfer->nrefs;
 344
 345         return xfer;
 346 }
 347
 348 struct pakfire_xfer* pakfire_xfer_unref(struct pakfire_xfer* xfer) {
 349         if (--xfer->nrefs > 0)
 350                 return xfer;
 351
 352         pakfire_xfer_free(xfer);
 353         return NULL;
 354 }
 355
 356 CURL* pakfire_xfer_handle(struct pakfire_xfer* xfer) {
 357         return xfer->handle;
 358 }
 359
 360 int pakfire_xfer_set_method(struct pakfire_xfer* xfer,
 361                 const pakfire_xfer_method_t method) {
 362         const char* m = NULL;
 363
 364         switch (method) {
 365                 case PAKFIRE_METHOD_DELETE:
 366                         m = "DELETE";
 367                         break;
 368
 369                 default:
 370                         return -EINVAL;
 371         }
 372
 373         return curl_easy_setopt(xfer->handle, CURLOPT_CUSTOMREQUEST, m);
 374 }
 375
 376
 377 const char* pakfire_xfer_get_title(struct pakfire_xfer* xfer) {
 378         char title[PATH_MAX];
 379         int r;
 380
 381         // Default to the filename if no title is set
 382         if (!*xfer->title) {
 383                 // Only use the basename
 384                 r = pakfire_path_basename(title, xfer->url);
 385                 if (r)
 386                         return NULL;
 387
 388                 // Store the title
 389                 r = pakfire_xfer_set_title(xfer, title);
 390                 if (r)
 391                         return NULL;
 392         }
 393
 394         return xfer->title;
 395 }
 396
 397 int pakfire_xfer_set_title(struct pakfire_xfer* xfer, const char* title) {
 398         return pakfire_string_set(xfer->title, title);
 399 }
 400
 401 int pakfire_xfer_set_baseurl(struct pakfire_xfer* xfer, const char* baseurl) {
 402         return pakfire_string_set(xfer->baseurl, baseurl);
 403 }
 404
 405 const char* pakfire_xfer_get_effective_url(struct pakfire_xfer* xfer) {
 406         return xfer->effective_url;
 407 }
 408
 409 int pakfire_xfer_set_mirrorlist(struct pakfire_xfer* xfer, struct pakfire_mirrorlist* mirrors) {
 410         if (xfer->mirrors)
 411                 pakfire_mirrorlist_unref(xfer->mirrors);
 412
 413         xfer->mirrors = pakfire_mirrorlist_ref(mirrors);
 414
 415         return 0;
 416 }
 417
 418 // Size
 419
 420 size_t pakfire_xfer_get_size(struct pakfire_xfer* xfer) {
 421         return xfer->expected_size;
 422 }
 423
 424 int pakfire_xfer_set_size(struct pakfire_xfer* xfer, size_t size) {
 425         xfer->expected_size = size;
 426
 427         return 0;
 428 }
 429
 430 int pakfire_xfer_verify_digest(struct pakfire_xfer* xfer, const enum pakfire_digest_types md,
 431                 const unsigned char* expected_digest, const size_t expected_digest_length) {
 432         // Check inputs
 433         if (!expected_digest || !expected_digest_length)
 434                 return -EINVAL;
 435
 436         // Expected digest length cannot be too long
 437         if (expected_digest_length > sizeof(xfer->expected_digest))
 438                 return -ENOBUFS;
 439
 440         // Store digest type
 441         switch (md) {
 442                 case PAKFIRE_DIGEST_SHA3_512:
 443                         xfer->md = EVP_sha3_512();
 444                         break;
 445
 446                 case PAKFIRE_DIGEST_SHA3_256:
 447                         xfer->md = EVP_sha3_256();
 448                         break;
 449
 450                 case PAKFIRE_DIGEST_BLAKE2B512:
 451                         xfer->md = EVP_blake2b512();
 452                         break;
 453
 454                 case PAKFIRE_DIGEST_BLAKE2S256:
 455                         xfer->md = EVP_blake2s256();
 456                         break;
 457
 458                 case PAKFIRE_DIGEST_SHA2_512:
 459                         xfer->md = EVP_sha512();
 460                         break;
 461
 462                 case PAKFIRE_DIGEST_SHA2_256:
 463                         xfer->md = EVP_sha256();
 464                         break;
 465
 466                 default:
 467                         return -ENOTSUP;
 468         }
 469
 470         // Store the expected digest and its length
 471         memcpy(xfer->expected_digest, expected_digest, expected_digest_length);
 472         xfer->expected_digest_length = expected_digest_length;
 473
 474         return 0;
 475 }
 476
 477 int pakfire_xfer_add_param(struct pakfire_xfer* xfer,
 478                 const char* key, const char* format, ...) {
 479         curl_mimepart* part = NULL;
 480         char* buffer = NULL;
 481         va_list args;
 482         int r;
 483
 484         // Allocate the MIME object if not done, yet
 485         if (!xfer->mime) {
 486                 xfer->mime = curl_mime_init(xfer->handle);
 487
 488                 if (!xfer->mime) {
 489                         CTX_ERROR(xfer->ctx, "Could not allocate the MIME object: %s\n",
 490                                 strerror(errno));
 491                         r = -errno;
 492                         goto ERROR;
 493                 }
 494         }
 495
 496         // Format value
 497         va_start(args, format);
 498         r = vasprintf(&buffer, format, args);
 499         va_end(args);
 500
 501         // Abort if we could not format the value
 502         if (r < 0)
 503                 goto ERROR;
 504
 505         // Allocate another MIME part
 506         part = curl_mime_addpart(xfer->mime);
 507         if (!part) {
 508                 CTX_ERROR(xfer->ctx, "Could not allocate MIME part: %s\n",
 509                         strerror(errno));
 510                 r = errno;
 511                 goto ERROR;
 512         }
 513
 514         // Set the key
 515         r = curl_mime_name(part, key);
 516         if (r) {
 517                 CTX_ERROR(xfer->ctx, "Could not set parameter key (%s): %s\n",
 518                         key, curl_easy_strerror(r));
 519                 goto ERROR;
 520         }
 521
 522         // Set the data
 523         r = curl_mime_data(part, buffer, CURL_ZERO_TERMINATED);
 524         if (r) {
 525                 CTX_ERROR(xfer->ctx, "Could not set parameter data (%s): %s\n",
 526                         key, curl_easy_strerror(r));
 527                 goto ERROR;
 528         }
 529
 530 ERROR:
 531         if (buffer)
 532                 free(buffer);
 533
 534         return r;
 535 }
 536
 537 static void pakfire_xfer_reset_output(struct pakfire_xfer* xfer) {
 538         if (xfer->fin) {
 539                 fclose(xfer->fin);
 540                 xfer->fin = NULL;
 541         }
 542 }
 543
 544 int pakfire_xfer_set_output(struct pakfire_xfer* xfer, FILE* f) {
 545         pakfire_xfer_reset_output(xfer);
 546
 547         // Store the new stream
 548         xfer->fin = f;
 549
 550         return 0;
 551 }
 552
 553 int pakfire_xfer_set_output_buffer(struct pakfire_xfer* xfer,
 554                 char** buffer, size_t* length) {
 555         FILE* f = NULL;
 556
 557         // Open a memory stream
 558         f = open_memstream(buffer, length);
 559         if (!f) {
 560                 CTX_ERROR(xfer->ctx, "Could not open memory stream: %s\n", strerror(errno));
 561                 return -errno;
 562         }
 563
 564         return pakfire_xfer_set_output(xfer, f);
 565 }
 566
 567 int pakfire_xfer_set_input(struct pakfire_xfer* xfer, FILE* f) {
 568         struct stat stat;
 569         int r;
 570
 571         // Fetch the file descriptor
 572         const int fd = fileno(f);
 573
 574         // Change direction
 575         xfer->direction = PAKFIRE_XFER_UPLOAD;
 576
 577         // Store the file handle
 578         xfer->fout = f;
 579
 580         // Try to find the upload size
 581         if (fd > 0) {
 582                 r = fstat(fd, &stat);
 583                 if (r)
 584                         return 0;
 585
 586                 // Store the expected filesize
 587                 xfer->expected_size = stat.st_size;
 588         }
 589
 590         return 0;
 591 }
 592
 593 static int pakfire_xfer_allocate_tmpfile(struct pakfire_xfer* xfer, const char* path) {
 594         char dirname[PATH_MAX];
 595         FILE* f = NULL;
 596         int fd = -1;
 597         int r;
 598
 599         // Find the directory name
 600         r = pakfire_path_dirname(dirname, path);
 601         if (r)
 602                 return r;
 603
 604         // Ensure the directory exists
 605         r = pakfire_mkdir(dirname, 0755);
 606         if (r)
 607                 return r;
 608
 609         // Open a new temporary file
 610         fd = open(dirname, O_TMPFILE|O_RDWR, 0600);
 611         if (fd < 0) {
 612                 CTX_ERROR(xfer->ctx, "Could not open temporary file in %s: %s\n",
 613                         dirname, strerror(errno));
 614                 return -errno;
 615         }
 616
 617         // Turn the file descriptor into a FILE handle
 618         f = fdopen(fd, "w+");
 619         if (!f)
 620                 return -errno;
 621
 622         // Set the handle as output
 623         return pakfire_xfer_set_output(xfer, f);
 624 }
 625
 626 int pakfire_xfer_set_output_path(struct pakfire_xfer* xfer, const char* path) {
 627         int r;
 628
 629         // Store the output path
 630         r = pakfire_string_set(xfer->path, path);
 631         if (r)
 632                 return r;
 633
 634         // Allocate a temporary file
 635         r = pakfire_xfer_allocate_tmpfile(xfer, path);
 636         if (r)
 637                 return r;
 638
 639         return 0;
 640 }
 641
 642 int pakfire_xfer_auth(struct pakfire_xfer* xfer) {
 643         // Enable authentication
 644         xfer->auth = 1;
 645
 646         return 0;
 647 }
 648
 649 static int pakfire_xfer_select_mirror(struct pakfire_xfer* xfer) {
 650         // Choose the next mirror
 651         if (xfer->mirror)
 652                 xfer->mirror = pakfire_mirrorlist_get_next(xfer->mirrors, xfer->mirror);
 653
 654         // If no mirror has been selected yet, choose the first one
 655         else
 656                 xfer->mirror = pakfire_mirrorlist_get_first(xfer->mirrors);
 657
 658         // Skip this mirror if it is broken
 659         while (xfer->mirror && pakfire_mirror_is_broken(xfer->mirror)) {
 660                 // Move on to the next mirror
 661                 xfer->mirror = pakfire_mirrorlist_get_next(xfer->mirrors, xfer->mirror);
 662         }
 663
 664         // No mirror found
 665         if (!xfer->mirror) {
 666                 CTX_ERROR(xfer->ctx, "No mirrors left to try\n");
 667
 668                 // No mirrors left
 669                 return ENOENT;
 670         }
 671
 672         CTX_DEBUG(xfer->ctx, "Selected mirror %s\n", pakfire_mirror_get_url(xfer->mirror));
 673
 674         return 0;
 675 }
 676
 677 static const char* curl_http_version(long v) {
 678         switch (v) {
 679 #ifdef CURL_HTTP_VERSION_3_0
 680                 case CURL_HTTP_VERSION_3_0:
 681                         return "HTTP/3.0";
 682 #endif
 683
 684                 case CURL_HTTP_VERSION_2_0:
 685                         return "HTTP/2.0";
 686
 687                 case CURL_HTTP_VERSION_1_1:
 688                         return "HTTP/1.1";
 689
 690                 case CURL_HTTP_VERSION_1_0:
 691                         return "HTTP/1.0";
 692         }
 693
 694         return "unknown";
 695 }
 696
 697 static int pakfire_xfer_save(struct pakfire_xfer* xfer) {
 698         struct utimbuf times = {
 699                 .actime  = 0,
 700                 .modtime = 0,
 701         };
 702         int r;
 703
 704         // Flush any buffered data out to disk
 705         if (xfer->fin)
 706                 fflush(xfer->fin);
 707
 708         // Nothing to do if path isn't set
 709         if (!*xfer->path)
 710                 return 0;
 711
 712         CTX_DEBUG(xfer->ctx, "Download successful. Storing result in %s\n", xfer->path);
 713
 714         int fd = fileno(xfer->fin);
 715
 716         // Make sure the parent directory exists
 717         r = pakfire_mkparentdir(xfer->path, 0755);
 718         if (r)
 719                 return r;
 720
 721         // Unlink the destination file (if it exists)
 722         unlink(xfer->path);
 723
 724         // Move the temporary file to its destination
 725         r = linkat(fd, "", AT_FDCWD, xfer->path, AT_EMPTY_PATH);
 726         if (r) {
 727                 CTX_ERROR(xfer->ctx, "Could not link destination file %s: %m\n",
 728                         xfer->path);
 729                 return r;
 730         }
 731
 732         // Filetime
 733         curl_easy_getinfo(xfer->handle, CURLINFO_FILETIME_T, &times.modtime);
 734
 735         if (times.modtime) {
 736                 r = utime(xfer->path, &times);
 737                 if (r)
 738                         CTX_ERROR(xfer->ctx, "Could not set mtime of %s: %m\n", xfer->path);
 739         }
 740
 741         return 0;
 742 }
 743
 744 int pakfire_xfer_fail(struct pakfire_xfer* xfer, int code) {
 745         int r;
 746
 747         CTX_DEBUG(xfer->ctx, "Xfer failed\n");
 748
 749         // Get file descriptor
 750         int fd = fileno(xfer->fin);
 751
 752         // Truncate downloaded data
 753         r = ftruncate(fd, 0);
 754         if (r)
 755                 return r;
 756
 757         // Did we use a mirror?
 758         if (xfer->mirror) {
 759                 pakfire_mirror_xfer_failed(xfer->mirror);
 760
 761                 // Try again with another mirror
 762                 return EAGAIN;
 763         }
 764
 765         return 0;
 766 }
 767
 768 int pakfire_xfer_done(struct pakfire_xfer* xfer, int code) {
 769         CURL* h = xfer->handle;
 770         int r;
 771         char* scheme = NULL;
 772         long response_code;
 773         long http_version;
 774         double total_time;
 775
 776         curl_off_t download_size = 0;
 777         curl_off_t download_speed = 0;
 778         curl_off_t upload_size = 0;
 779         curl_off_t upload_speed = 0;
 780
 781         // Finish progress
 782         r = pakfire_progress_finish(xfer->progress);
 783         if (r)
 784                 return r;
 785
 786         CTX_DEBUG(xfer->ctx, "cURL xfer done: %d - %s\n",
 787                 code, curl_easy_strerror(code));
 788
 789         // Finish message digest computation
 790         if (xfer->evp) {
 791                 r = EVP_DigestFinal_ex(xfer->evp, xfer->computed_digest, &xfer->computed_digest_length);
 792                 if (r != 1) {
 793                         CTX_ERROR(xfer->ctx, "Could not finish message digest computation: %s\n",
 794                                 ERR_error_string(ERR_get_error(), NULL));
 795                         return 1;
 796                 }
 797         }
 798
 799         // Protocol
 800         curl_easy_getinfo(h, CURLINFO_SCHEME, &scheme);
 801
 802         // Effective URL
 803         curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &xfer->effective_url);
 804         if (xfer->effective_url)
 805                 CTX_DEBUG(xfer->ctx, "  Effective URL: %s\n", xfer->effective_url);
 806
 807         // Response code
 808         curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &response_code);
 809         if (response_code)
 810                 CTX_DEBUG(xfer->ctx, "  Response code: %ld\n", response_code);
 811
 812         // HTTP Version
 813         curl_easy_getinfo(h, CURLINFO_HTTP_VERSION, &http_version);
 814         if (http_version)
 815                 CTX_DEBUG(xfer->ctx, "  HTTP Version: %s\n", curl_http_version(http_version));
 816
 817         // Total Times
 818         curl_easy_getinfo(h, CURLINFO_TOTAL_TIME, &total_time);
 819         CTX_DEBUG(xfer->ctx, "  Total Time: %.2fs\n", total_time);
 820
 821         // Download Size
 822         r = curl_easy_getinfo(h, CURLINFO_SIZE_DOWNLOAD_T, &download_size);
 823         if (r)
 824                 return r;
 825
 826         if (download_size)
 827                 CTX_DEBUG(xfer->ctx, "  Download Size: %ld bytes\n", download_size);
 828
 829         // Download Speed
 830         r = curl_easy_getinfo(h, CURLINFO_SPEED_DOWNLOAD_T, &download_speed);
 831         if (r)
 832                 return r;
 833
 834         if (download_speed)
 835                 CTX_DEBUG(xfer->ctx, "  Download Speed: %ld bps\n", download_speed);
 836
 837         // Upload Size
 838         r = curl_easy_getinfo(h, CURLINFO_SIZE_UPLOAD_T, &upload_size);
 839         if (r)
 840                 return r;
 841
 842         if (upload_size)
 843                 CTX_DEBUG(xfer->ctx, "  Upload Size: %ld bytes\n", upload_size);
 844
 845         // Upload Speed
 846         r = curl_easy_getinfo(h, CURLINFO_SPEED_UPLOAD_T, &upload_speed);
 847         if (r)
 848                 return r;
 849
 850         if (upload_speed)
 851                 CTX_DEBUG(xfer->ctx, "  Upload Speed: %ld bps\n", upload_speed);
 852
 853         // Message Digest
 854         char* hexdigest = __pakfire_hexlify(xfer->computed_digest, xfer->computed_digest_length);
 855         if (hexdigest && *hexdigest) {
 856                 CTX_DEBUG(xfer->ctx, "  Message Digest: %s\n", hexdigest);
 857                 free(hexdigest);
 858         }
 859
 860         // Check if digests match
 861         if (xfer->evp) {
 862                 r = CRYPTO_memcmp(xfer->computed_digest, xfer->expected_digest,
 863                         xfer->computed_digest_length);
 864
 865                 // If they don't match, log an error and try again
 866                 if (r) {
 867                         char* computed_hexdigest = __pakfire_hexlify(xfer->computed_digest,
 868                                 xfer->computed_digest_length);
 869                         char* expected_hexdigest = __pakfire_hexlify(xfer->expected_digest,
 870                                 xfer->expected_digest_length);
 871
 872                         CTX_ERROR(xfer->ctx, "Download checksum for %s didn't match:\n", xfer->effective_url);
 873                         CTX_ERROR(xfer->ctx, "  Expected: %s\n", expected_hexdigest);
 874                         CTX_ERROR(xfer->ctx, "  Computed: %s\n", computed_hexdigest);
 875
 876                         if (computed_hexdigest)
 877                                 free(computed_hexdigest);
 878                         if (expected_hexdigest)
 879                                 free(expected_hexdigest);
 880
 881                         // Make this download fail
 882                         r = pakfire_xfer_fail(xfer, 0);
 883                         if (r)
 884                                 return r;
 885
 886                         return 1;
 887                 }
 888         }
 889
 890         // If we could not determine the scheme...
 891         if (!scheme)
 892                 r = pakfire_xfer_fail(xfer, 0);
 893
 894         // FILE
 895         else if (strcmp(scheme, "FILE") == 0) {
 896                 // Handle any errors
 897                 if (code)
 898                         r = pakfire_xfer_fail(xfer, code);
 899                 else
 900                         r = pakfire_xfer_save(xfer);
 901
 902                 return r;
 903
 904         // HTTPS + HTTP
 905         } else if ((strcmp(scheme, "HTTPS") == 0) || (strcmp(scheme, "HTTP") == 0)) {
 906                 switch (response_code) {
 907                         // 200 - OK
 908                         case 200:
 909                                 r = pakfire_xfer_save(xfer);
 910                                 if (r)
 911                                         return r;
 912                                 break;
 913
 914                         // Treat all other response codes as an error
 915                         default:
 916                                 r = pakfire_xfer_fail(xfer, code);
 917                                 if (r)
 918                                         return r;
 919
 920                                 // Error
 921                                 return 1;
 922                 }
 923
 924         // FTP
 925         } else if (strcmp(scheme, "FTP") == 0) {
 926                 if (response_code == 226)
 927                         r = pakfire_xfer_save(xfer);
 928                 else
 929                         r = pakfire_xfer_fail(xfer, code);
 930
 931                 return r;
 932         }
 933
 934         // Success
 935         return 0;
 936 }
 937
 938 static int pakfire_xfer_update(void* data,
 939                 curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
 940         struct pakfire_xfer* xfer = data;
 941
 942         switch (xfer->direction) {
 943                 case PAKFIRE_XFER_DOWNLOAD:
 944                         // Update the expected size
 945                         xfer->expected_size = dltotal;
 946
 947                         // Update the xferred counter
 948                         xfer->xferred = dlnow;
 949                         break;
 950
 951                 case PAKFIRE_XFER_UPLOAD:
 952                         // Update the expected size
 953                         xfer->expected_size = ultotal;
 954
 955                         // Update the xferred counter
 956                         xfer->xferred = ulnow;
 957                         break;
 958         }
 959
 960         // Do nothing if no progress indicator has been set up
 961         if (!xfer->progress)
 962                 return 0;
 963
 964         // Set maximum (because this might have changed)
 965         pakfire_progress_set_max_value(xfer->progress, xfer->expected_size);
 966
 967         // Update current value
 968         return pakfire_progress_update(xfer->progress, xfer->xferred);
 969 }
 970
 971 static int pakfire_xfer_prepare_progress(struct pakfire_xfer* xfer,
 972                 struct pakfire_progress* parent, int flags) {
 973         const char* title = NULL;
 974         int progress_flags =
 975                 PAKFIRE_PROGRESS_SHOW_PERCENTAGE |
 976                 PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED |
 977                 PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED |
 978                 PAKFIRE_PROGRESS_SHOW_ETA;
 979         int r;
 980
 981         // If this has already been set up, we skip it
 982         if (xfer->progress)
 983                 return 0;
 984
 985         // Show no progress if requested
 986         if (flags & PAKFIRE_XFER_NO_PROGRESS)
 987                 progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
 988
 989         // Show no progress if we have a parent progress
 990         else if (parent)
 991                 progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
 992
 993         // Make a new progress meter
 994         r = pakfire_progress_create(&xfer->progress, xfer->ctx, progress_flags, parent);
 995         if (r)
 996                 return r;
 997
 998         // Set the title
 999         title = pakfire_xfer_get_title(xfer);
1000         if (title) {
1001                 r = pakfire_progress_set_title(xfer->progress, title);
1002                 if (r)
1003                         return r;
1004         }
1005
1006         // Configure callbacks
1007         curl_easy_setopt(xfer->handle, CURLOPT_XFERINFOFUNCTION,
1008                 pakfire_xfer_update);
1009         curl_easy_setopt(xfer->handle, CURLOPT_XFERINFODATA, xfer);
1010         curl_easy_setopt(xfer->handle, CURLOPT_NOPROGRESS, 0L);
1011
1012         // Start progress
1013         r = pakfire_progress_start(xfer->progress, xfer->expected_size);
1014         if (r)
1015                 return r;
1016
1017         return 0;
1018 }
1019
1020 static int pakfire_xfer_prepare_url(struct pakfire_xfer* xfer) {
1021         int r;
1022
1023         // Simply set absolute URLs
1024         if (pakfire_string_is_url(xfer->url)) {
1025                 r = curl_url_set(xfer->fullurl, CURLUPART_URL, xfer->url, 0);
1026                 if (r)
1027                         goto ERROR;
1028
1029         // Join path if we are using mirrors
1030         } else if (xfer->mirrors && !pakfire_mirrorlist_empty(xfer->mirrors)) {
1031                 r = pakfire_xfer_select_mirror(xfer);
1032                 if (r)
1033                         goto ERROR;
1034
1035                 // Set the mirror's base URL first
1036                 r = curl_url_set(xfer->fullurl, CURLUPART_URL,
1037                         pakfire_mirror_get_url(xfer->mirror), 0);
1038                 if (r)
1039                         goto ERROR;
1040
1041                 // Then append our own part
1042                 r = curl_url_set(xfer->fullurl, CURLUPART_URL, xfer->url, 0);
1043                 if (r)
1044                         goto ERROR;
1045
1046         // Use baseurl
1047         } else if (*xfer->baseurl) {
1048                 // Set the base URL first
1049                 r = curl_url_set(xfer->fullurl, CURLUPART_URL, xfer->baseurl, 0);
1050                 if (r)
1051                         goto ERROR;
1052
1053                 // Then append our own part
1054                 r = curl_url_set(xfer->fullurl, CURLUPART_URL, xfer->url, 0);
1055                 if (r)
1056                         goto ERROR;
1057
1058         // Fail if we could not set the URL
1059         } else {
1060                 CTX_ERROR(xfer->ctx, "Invalid xfer %s\n", xfer->url);
1061                 r = -EINVAL;
1062                 goto ERROR;
1063         }
1064
1065         // Set the URL
1066         r = curl_easy_setopt(xfer->handle, CURLOPT_CURLU, xfer->fullurl);
1067         if (r) {
1068                 CTX_ERROR(xfer->ctx, "Could not set the URL: %s\n", curl_easy_strerror(r));
1069                 goto ERROR;
1070         }
1071
1072 ERROR:
1073         return r;
1074 }
1075
1076 int pakfire_xfer_prepare(struct pakfire_xfer* xfer, struct pakfire_progress* progress, int flags) {
1077         int r;
1078
1079         // Increment tries
1080         xfer->tries++;
1081
1082         // Set special options for direction
1083         switch (xfer->direction) {
1084                 case PAKFIRE_XFER_DOWNLOAD:
1085                         break;
1086
1087                 case PAKFIRE_XFER_UPLOAD:
1088                         // Let cURL know that we are uploading things
1089                         r = curl_easy_setopt(xfer->handle, CURLOPT_UPLOAD, 1L);
1090                         if (r) {
1091                                 CTX_ERROR(xfer->ctx, "Could not enable upload\n");
1092                                 return r;
1093                         }
1094
1095                         // Tell it the expected upload size
1096                         if (xfer->expected_size) {
1097                                 r = curl_easy_setopt(xfer->handle,
1098                                         CURLOPT_INFILESIZE_LARGE, (curl_off_t)xfer->expected_size);
1099                                 if (r) {
1100                                         CTX_ERROR(xfer->ctx, "Could not set upload size\n");
1101                                         return r;
1102                                 }
1103                         }
1104
1105                         // Upload files chunked
1106                         xfer->headers = curl_slist_append(xfer->headers, "Transfer-Encoding: chunked");
1107                         break;
1108         }
1109
1110         // Compose the URL
1111         r = pakfire_xfer_prepare_url(xfer);
1112         if (r) {
1113                 CTX_ERROR(xfer->ctx, "Could not compose URL: %m\n");
1114                 return r;
1115         }
1116
1117         // Add any headers
1118         if (xfer->headers) {
1119                 r = curl_easy_setopt(xfer->handle, CURLOPT_HTTPHEADER, xfer->headers);
1120                 if (r) {
1121                         CTX_ERROR(xfer->ctx, "Could not set headers: %s\n", curl_easy_strerror(r));
1122                         return r;
1123                 }
1124         }
1125
1126         // Add any payload
1127         if (xfer->mime) {
1128                 r = curl_easy_setopt(xfer->handle, CURLOPT_MIMEPOST, xfer->mime);
1129                 if (r) {
1130                         CTX_ERROR(xfer->ctx, "Could not set POST payload: %s\n", curl_easy_strerror(r));
1131                         return r;
1132                 }
1133         }
1134
1135         // Authentication
1136         if (xfer->auth) {
1137                 // Request SPNEGO
1138                 r = curl_easy_setopt(xfer->handle, CURLOPT_HTTPAUTH, CURLAUTH_NEGOTIATE|CURLAUTH_ONLY);
1139                 if (r) {
1140                         CTX_ERROR(xfer->ctx, "Could not enable SPNEGO\n");
1141                         return r;
1142                 }
1143
1144                 // Set an empty username
1145                 r = curl_easy_setopt(xfer->handle, CURLOPT_USERPWD, ":");
1146                 if (r) {
1147                         CTX_ERROR(xfer->ctx, "Could not set username\n");
1148                         return r;
1149                 }
1150         }
1151
1152         // Drop any previously used EVP contexts
1153         if (xfer->evp) {
1154                 EVP_MD_CTX_free(xfer->evp);
1155                 xfer->evp = NULL;
1156         }
1157
1158         // Create a new EVP context
1159         if (xfer->md) {
1160                 xfer->evp = EVP_MD_CTX_new();
1161                 if (!xfer->evp) {
1162                         CTX_ERROR(xfer->ctx, "Could not create EVP context: %m\n");
1163                         return 1;
1164                 }
1165
1166                 // Initialize the EVP context
1167                 r = EVP_DigestInit_ex(xfer->evp, xfer->md, NULL);
1168                 if (r != 1) {
1169                         CTX_ERROR(xfer->ctx, "Could not initialize EVP context: %s\n",
1170                                 ERR_error_string(ERR_get_error(), NULL));
1171                         return 1;
1172                 }
1173         }
1174
1175         // Setup progress
1176         r = pakfire_xfer_prepare_progress(xfer, progress, flags);
1177         if (r)
1178                 return r;
1179
1180         return 0;
1181 }
1182
1183 int pakfire_xfer_run(struct pakfire_xfer* xfer, int flags) {
1184         int r;
1185
1186 AGAIN:
1187         // Prepare the xfer
1188         r = pakfire_xfer_prepare(xfer, NULL, flags);
1189         if (r) {
1190                 CTX_ERROR(xfer->ctx, "Could not prepare xfer %s: %s\n",
1191                         xfer->url, strerror(-r));
1192                 return r;
1193         }
1194
1195         // Perform the action
1196         r = curl_easy_perform(xfer->handle);
1197
1198         // Handle the result
1199         r = pakfire_xfer_done(xfer, r);
1200
1201         // Repeat the xfer if asked
1202         switch (-r) {
1203                 case EAGAIN:
1204                         goto AGAIN;
1205
1206                 default:
1207                         break;
1208         }
1209
1210         return r;
1211 }
1212
1213 static int pakfire_xfer_handle_api_error(
1214                 struct pakfire_xfer* xfer, const struct json_object* error) {
1215         struct json_object* message = NULL;
1216         struct json_object* code = NULL;
1217         const char* m = NULL;
1218         unsigned int c = 0;
1219
1220         // Fetch the URL
1221         const char* url = pakfire_xfer_get_effective_url(xfer);
1222
1223         // Fetch the code
1224         if (!json_object_object_get_ex(error, "code", &code))
1225                 return -EBADMSG;
1226
1227         // Check if the code is an integer
1228         if (!json_object_is_type(code, json_type_int))
1229                 return -EBADMSG;
1230
1231         // Fetch the message
1232         if (!json_object_object_get_ex(error, "message", &message))
1233                 return -EBADMSG;
1234
1235         // Check if the message is a string
1236         if (!json_object_is_type(message, json_type_string))
1237                 return -EBADMSG;
1238
1239         c = json_object_get_uint64(code);
1240         m = json_object_get_string(message);
1241
1242         // Log the error
1243         CTX_ERROR(xfer->ctx, "%s responded with error %u (%s):\n  %s\n",
1244                 url, c, strerror(c), m);
1245
1246         return -c;
1247 }
1248
1249 /*
1250         This function parses an API response
1251 */
1252 static int pakfire_xfer_parse_api_response(struct pakfire_xfer* xfer,
1253                 const char* buffer, const size_t length, struct json_object** object) {
1254         struct json_object* error = NULL;
1255         struct json_object* o = NULL;
1256         int r;
1257
1258         // Check if we received any data
1259         if (!length) {
1260                 CTX_ERROR(xfer->ctx, "Received an empty response\n");
1261                 r = -EBADMSG;
1262                 goto ERROR;
1263         }
1264
1265         // XXX Maybe fetch the parser's error message here?!
1266
1267         // Parse the buffer
1268         o = pakfire_json_parse(xfer->ctx, buffer, length);
1269         if (!o) {
1270                 CTX_ERROR(xfer->ctx, "Could not parse the response\n");
1271                 r = -EBADMSG;
1272                 goto ERROR;
1273         }
1274
1275         // Check if the response is a dictionary
1276         if (!json_object_is_type(o, json_type_object)) {
1277                 CTX_ERROR(xfer->ctx, "The received object is not a JSON dict\n");
1278                 r = -EBADMSG;
1279                 goto ERROR;
1280         }
1281
1282         // Fetch error
1283         r = json_object_object_get_ex(o, "error", &error);
1284         if (r) {
1285                 r = pakfire_xfer_handle_api_error(xfer, error);
1286                 goto ERROR;
1287         }
1288
1289         // Return the object
1290         if (object)
1291                 *object = json_object_get(o);
1292
1293 ERROR:
1294         if (o)
1295                 json_object_put(o);
1296
1297         return r;
1298 }
1299
1300 static int pakfire_xfer_run_api_request_once(struct pakfire_xfer* xfer, struct json_object** response) {
1301         char* buffer = NULL;
1302         size_t length = 0;
1303         int r;
1304
1305         // Write the response to the buffer
1306         r = pakfire_xfer_set_output_buffer(xfer, &buffer, &length);
1307         if (r)
1308                 goto ERROR;
1309
1310         // Run the xfer
1311         r = pakfire_xfer_run(xfer, PAKFIRE_XFER_NO_PROGRESS);
1312         if (r)
1313                 goto ERROR;
1314
1315         // Parse the response
1316         r = pakfire_xfer_parse_api_response(xfer, buffer, length, response);
1317         if (r) {
1318                 CTX_ERROR(xfer->ctx, "Could not parse the API response: %s\n", strerror(-r));
1319                 goto ERROR;
1320         }
1321
1322 ERROR:
1323         // Reset the output stream
1324         pakfire_xfer_reset_output(xfer);
1325
1326         if (buffer)
1327                 free(buffer);
1328
1329         return r;
1330 }
1331
1332 /*
1333         This function sends a request and automatically parses the response.
1334         The response might optionally be returned if response is not NULL.
1335 */
1336 int pakfire_xfer_run_api_request(struct pakfire_xfer* xfer, struct json_object** response) {
1337         int r;
1338
1339         // Loop indefinitely...
1340         for (;;) {
1341                 r = pakfire_xfer_run_api_request_once(xfer, response);
1342                 switch (r) {
1343                         // XXX need to catch errors and act accordingly
1344
1345                         default:
1346                                 return r;
1347                 }
1348         }
1349
1350         return 0;
1351 }