From: Michael Tremer Date: Mon, 2 Oct 2023 10:17:36 +0000 (+0000) Subject: downloader: This is the beginning of a rather large rewrite X-Git-Tag: 0.9.30~1571 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f31b9f876c2ddc8551bb522e57491a5652ae0713;p=pakfire.git downloader: This is the beginning of a rather large rewrite The downloader code has become very complicated because we have large functions that perform everything at once. This patch splits a lot of functionality into smaller steps which are easier to handle, allow better expansion and so on. You know? Signed-off-by: Michael Tremer --- diff --git a/src/libpakfire/downloader.c b/src/libpakfire/downloader.c index 2e4532a1b..abb0d9348 100644 --- a/src/libpakfire/downloader.c +++ b/src/libpakfire/downloader.c @@ -19,6 +19,7 @@ #############################################################################*/ #include +#include #include #include #include @@ -54,9 +55,18 @@ struct pakfire_mirror { }; struct pakfire_transfer { + struct pakfire* pakfire; + int nrefs; + + // Reference to the downloader struct pakfire_downloader* downloader; + // Reference to the progress indicator + struct pakfire_progress* progress; + TAILQ_ENTRY(pakfire_transfer) nodes; + + // cURL handle CURL* handle; char title[NAME_MAX]; @@ -65,6 +75,10 @@ struct pakfire_transfer { enum pakfire_transfer_flags flags; int tries; + // Size + size_t expected_size; + size_t transferred; + // Temporary file char tempfile[PATH_MAX]; FILE* f; @@ -87,22 +101,23 @@ struct pakfire_downloader { struct pakfire* pakfire; int nrefs; - struct pakfire_progress* progress; +// struct pakfire_progress* progress; unsigned int parallel; // cURL multi handle CURLM* curl; - TAILQ_HEAD(transfers, pakfire_transfer) transfers; -}; -static char* pakfire_url_join(const char* part1, const char* part2) { - char* url = NULL; + // Transfers + TAILQ_HEAD(transfers_queued, pakfire_transfer) transfers_queued; + TAILQ_HEAD(transfers_running, pakfire_transfer) transfers_running; + TAILQ_HEAD(transfers_finished, pakfire_transfer) transfers_finished; +}; - int r = asprintf(&url, "%s/%s", part1, part2); - if (r < 0) - return NULL; +#define pakfire_url_join(s, part1, part2) \ + __pakfire_url_join(s, sizeof(s), part1, part2) - return url; +static int __pakfire_url_join(char* s, const size_t length, const char* part1, const char* part2) { + return __pakfire_string_format(s, length, "%s/%s", part1, part2); } static int pakfire_downloader_setup_curl(struct pakfire_downloader* downloader) { @@ -128,7 +143,7 @@ static int pakfire_downloader_setup_curl(struct pakfire_downloader* downloader) return 0; } -static void pakfire_transfer_free(struct pakfire_transfer* transfer) { +static void pakfire_downloader_transfer_free(struct pakfire_transfer* transfer) { if (transfer->handle) curl_easy_cleanup(transfer->handle); @@ -146,21 +161,40 @@ static void pakfire_transfer_free(struct pakfire_transfer* transfer) { if (transfer->mirrors) pakfire_mirrorlist_unref(transfer->mirrors); + if (transfer->progress) + pakfire_progress_unref(transfer->progress); + if (transfer->pakfire) + pakfire_unref(transfer->pakfire); free(transfer); } static void pakfire_downloader_free(struct pakfire_downloader* downloader) { - // Free any unprocessed transfers - struct pakfire_transfer* transfer; - while (!TAILQ_EMPTY(&downloader->transfers)) { - transfer = TAILQ_LAST(&downloader->transfers, transfers); - TAILQ_REMOVE(&downloader->transfers, transfer, nodes); - pakfire_transfer_free(transfer); + struct pakfire_transfer* transfer = NULL; + + // Free any queued transfers + while (!TAILQ_EMPTY(&downloader->transfers_queued)) { + transfer = TAILQ_LAST(&downloader->transfers_queued, transfers_queued); + TAILQ_REMOVE(&downloader->transfers_queued, transfer, nodes); + + pakfire_downloader_transfer_unref(transfer); } - if (downloader->progress) - pakfire_progress_unref(downloader->progress); + // Free any running transfers + while (!TAILQ_EMPTY(&downloader->transfers_running)) { + transfer = TAILQ_LAST(&downloader->transfers_running, transfers_running); + TAILQ_REMOVE(&downloader->transfers_running, transfer, nodes); + + pakfire_downloader_transfer_unref(transfer); + } + + // Free any finished transfers + while (!TAILQ_EMPTY(&downloader->transfers_finished)) { + transfer = TAILQ_LAST(&downloader->transfers_finished, transfers_finished); + TAILQ_REMOVE(&downloader->transfers_finished, transfer, nodes); + + pakfire_downloader_transfer_unref(transfer); + } if (downloader->curl) curl_multi_cleanup(downloader->curl); @@ -170,18 +204,19 @@ static void pakfire_downloader_free(struct pakfire_downloader* downloader) { } int pakfire_downloader_create(struct pakfire_downloader** downloader, struct pakfire* pakfire) { + struct pakfire_downloader* d = NULL; int r; // Fail if pakfire is running in offline mode if (pakfire_has_flag(pakfire, PAKFIRE_FLAGS_OFFLINE)) { ERROR(pakfire, "Cannot initialize downloader in offline mode\n"); - errno = ENOTSUP; - return 1; + return -ENOTSUP; } - struct pakfire_downloader* d = calloc(1, sizeof(*d)); + // Allocate a new object + d = calloc(1, sizeof(*d)); if (!d) - return ENOMEM; + return -ENOMEM; // Store reference to Pakfire d->pakfire = pakfire_ref(pakfire); @@ -192,23 +227,17 @@ int pakfire_downloader_create(struct pakfire_downloader** downloader, struct pak // Set parallelism d->parallel = MAX_PARALLEL; - // Init transfers queue - TAILQ_INIT(&d->transfers); + // Init transfer queues + TAILQ_INIT(&d->transfers_queued); + TAILQ_INIT(&d->transfers_running); + TAILQ_INIT(&d->transfers_finished); // Setup cURL r = pakfire_downloader_setup_curl(d); if (r) goto ERROR; - // Create the progress indicator - r = pakfire_progress_create(&d->progress, d->pakfire, - PAKFIRE_PROGRESS_SHOW_PERCENTAGE | - PAKFIRE_PROGRESS_SHOW_ETA | - PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED | - PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED); - if (r) - goto ERROR; - + // Success *downloader = d; return 0; @@ -261,6 +290,255 @@ static int debug_callback(CURL *handle, curl_infotype type, } #endif +static size_t pakfire_downloader_transfer_write( + char* data, size_t size, size_t nmemb, void* p) { + struct pakfire_transfer* transfer = p; + int r; + + // Do not write empty blocks + if (!nmemb) + return nmemb; + + // Update message digest + if (transfer->evp) { + r = EVP_DigestUpdate(transfer->evp, data, nmemb); + if (r != 1) { + ERROR(transfer->downloader->pakfire, "EVP_DigestUpdate failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + return 0; + } + } + + // Write everything to the allocated file descriptor + return fwrite(data, size, nmemb, transfer->f); +} + +static int pakfire_downloader_transfer_setup( + struct pakfire_downloader* downloader, struct pakfire_transfer* transfer) { + struct pakfire_config* config = NULL; + const char* proxy = NULL; + int r; + + // Fetch global configuration + config = pakfire_get_config(downloader->pakfire); + + // Set global configuration + if (config) { + proxy = pakfire_config_get(config, "general", "proxy", NULL); + if (proxy) + curl_easy_setopt(transfer->handle, CURLOPT_PROXY, proxy); + } + + // Be a good net citizen and set a user agent + curl_easy_setopt(transfer->handle, CURLOPT_USERAGENT, PACKAGE_NAME "/" PACKAGE_VERSION); + +#ifdef ENABLE_DEBUG + // Enable logging/debugging + curl_easy_setopt(transfer->handle, CURLOPT_VERBOSE, 1L); + + curl_easy_setopt(transfer->handle, CURLOPT_DEBUGFUNCTION, debug_callback); + curl_easy_setopt(transfer->handle, CURLOPT_DEBUGDATA, downloader->pakfire); +#endif + + // Limit protocols to HTTPS, HTTP, FTP and FILE + curl_easy_setopt(transfer->handle, CURLOPT_PROTOCOLS_STR, "HTTPS,HTTP,FTP,FILE"); + + // Reference back to this transfer + curl_easy_setopt(transfer->handle, CURLOPT_PRIVATE, transfer); + + // Follow any redirects + curl_easy_setopt(transfer->handle, CURLOPT_FOLLOWLOCATION, 1); + + // Write all data to the callback function + curl_easy_setopt(transfer->handle, + CURLOPT_WRITEFUNCTION, pakfire_downloader_transfer_write); + curl_easy_setopt(transfer->handle, CURLOPT_WRITEDATA, transfer); + + // Success + r = 0; + + // Cleanup + if (config) + pakfire_config_unref(config); + + return r; +} + +int pakfire_downloader_transfer_create(struct pakfire_transfer** transfer, + struct pakfire_downloader* downloader, const char* url) { + struct pakfire_transfer* t = NULL; + int r; + + // Allocate a new transfer + t = calloc(1, sizeof(*t)); + if (!t) + return -errno; + + // Store a reference to Pakfire + t->pakfire = pakfire_ref(downloader->pakfire); + + // Initialize the reference counter + t->nrefs = 1; + + // Store a reference to the downloader + t->downloader = pakfire_downloader_ref(downloader); + + // Store the URL + r = pakfire_string_set(t->url, url); + if (r) + goto ERROR; + + // Allocate a handle + t->handle = curl_easy_init(); + if (!t->handle) { + r = 1; + goto ERROR; + } + + // Setup the transfer + r = pakfire_downloader_transfer_setup(downloader, t); + if (r) + goto ERROR; + + // Push this transfer onto the queue + TAILQ_INSERT_HEAD(&downloader->transfers_queued, + pakfire_downloader_transfer_ref(t), nodes); + + // Return the reference + *transfer = t; + + return 0; + +ERROR: + if (t) + pakfire_downloader_transfer_unref(t); + + return r; +} + +struct pakfire_transfer* pakfire_downloader_transfer_ref(struct pakfire_transfer* transfer) { + ++transfer->nrefs; + + return transfer; +} + +struct pakfire_transfer* pakfire_downloader_transfer_unref(struct pakfire_transfer* transfer) { + if (--transfer->nrefs > 0) + return transfer; + + pakfire_downloader_transfer_free(transfer); + return NULL; +} + +const char* pakfire_downloader_transfer_get_title(struct pakfire_transfer* transfer) { + char path[PATH_MAX]; + int r; + + // Default to the filename if no title is set + if (!*transfer->title) { + r = pakfire_basename(path, transfer->url); + if (r) + return NULL; + + // Store the title + r = pakfire_downloader_transfer_set_title(transfer, path); + if (r) + return NULL; + } + + return transfer->title; +} + +int pakfire_downloader_transfer_set_title( + struct pakfire_transfer* transfer, const char* title) { + return pakfire_string_set(transfer->title, title); +} + +int pakfire_downloader_transfer_set_baseurl( + struct pakfire_transfer* transfer, const char* baseurl) { + return pakfire_string_set(transfer->baseurl, baseurl); +} + +int pakfire_downloader_transfer_set_mirrorlist( + struct pakfire_transfer* transfer, struct pakfire_mirrorlist* mirrors) { + if (transfer->mirrors) + pakfire_mirrorlist_unref(transfer->mirrors); + + transfer->mirrors = pakfire_mirrorlist_ref(mirrors); + + return 0; +} + +static size_t pakfire_downloader_transfer_get_size(struct pakfire_transfer* transfer) { + return transfer->expected_size; +} + +int pakfire_downloader_transfer_set_size( + struct pakfire_transfer* transfer, size_t size) { + transfer->expected_size = size; + + return 0; +} + +int pakfire_downloader_transfer_verify_digest( + struct pakfire_transfer* transfer, const enum pakfire_digest_types md, + const unsigned char* expected_digest, const size_t expected_digest_length) { + // Check inputs + if (!expected_digest || !expected_digest_length) + return -EINVAL; + + // Expected digest length cannot be too long + if (expected_digest_length > sizeof(transfer->expected_digest)) + return -ENOBUFS; + + // Store digest type + switch (md) { + case PAKFIRE_DIGEST_SHA3_512: + transfer->md = EVP_sha3_512(); + break; + + case PAKFIRE_DIGEST_SHA3_256: + transfer->md = EVP_sha3_256(); + break; + + case PAKFIRE_DIGEST_BLAKE2B512: + transfer->md = EVP_blake2b512(); + break; + + case PAKFIRE_DIGEST_BLAKE2S256: + transfer->md = EVP_blake2s256(); + break; + + case PAKFIRE_DIGEST_SHA2_512: + transfer->md = EVP_sha512(); + break; + + case PAKFIRE_DIGEST_SHA2_256: + transfer->md = EVP_sha256(); + break; + + default: + return -ENOTSUP; + } + + // Store the expected digest and its length + memcpy(transfer->expected_digest, expected_digest, expected_digest_length); + transfer->expected_digest_length = expected_digest_length; + + return 0; +} + +int pakfire_downloader_transfer_set_output(struct pakfire_transfer* transfer, FILE* f) { + transfer->f = f; + + return 0; +} + +int pakfire_downloader_transfer_set_target( + struct pakfire_transfer* transfer, const char* path) { + return pakfire_string_set(transfer->path, path); +} + static struct pakfire_transfer* pakfire_downloader_create_transfer( struct pakfire_downloader* downloader, const char* baseurl, @@ -429,33 +707,11 @@ static struct pakfire_transfer* pakfire_downloader_create_transfer( return transfer; ERROR: - pakfire_transfer_free(transfer); + pakfire_downloader_transfer_free(transfer); return NULL; } -int pakfire_downloader_add_transfer( - struct pakfire_downloader* downloader, - const char* baseurl, - struct pakfire_mirrorlist* mirrors, - const char* title, - const char* url, - const char* path, - enum pakfire_digest_types md, - const unsigned char* expected_digest, - const size_t expected_digest_length, - enum pakfire_transfer_flags flags) { - struct pakfire_transfer* transfer = pakfire_downloader_create_transfer( - downloader, baseurl, mirrors, title, url, path, md, expected_digest, expected_digest_length, flags); - if (!transfer) - return 1; - - // Push this transfer onto the queue - TAILQ_INSERT_HEAD(&downloader->transfers, transfer, nodes); - - return 0; -} - static int pakfire_transfer_select_mirror(struct pakfire_downloader* downloader, struct pakfire_transfer* transfer) { // Choose the next mirror @@ -510,20 +766,30 @@ static int pakfire_transfer_save(struct pakfire_downloader* downloader, struct utimbuf times; int r; + // Nothing to do if path isn't set + if (!*transfer->path) + return 0; + DEBUG(downloader->pakfire, "Download successful. Storing result in %s\n", transfer->path); - if (!(transfer->flags & PAKFIRE_TRANSFER_NOTEMP)) { - // Remove destination (if it exists) - unlink(transfer->path); + // Flush any buffered data out to disk + fflush(transfer->f); - // Move the temporary file to its destination - r = link(transfer->tempfile, transfer->path); - if (r) { - ERROR(downloader->pakfire, "Could not link destination file %s: %m\n", - transfer->path); - return r; - } + // Make sure the parent directory exists + r = pakfire_mkparentdir(transfer->path, 0755); + if (r) + return r; + + // Unlink the destination file (if it exists) + unlink(transfer->path); + + // Move the temporary file to its destination + r = link(transfer->tempfile, transfer->path); + if (r) { + ERROR(downloader->pakfire, "Could not link destination file %s: %m\n", + transfer->path); + return r; } // Filetime @@ -577,6 +843,11 @@ static int pakfire_transfer_done(struct pakfire_downloader* downloader, curl_off_t download_size; curl_off_t speed; + // Finish progress + r = pakfire_progress_finish(transfer->progress); + if (r) + return r; + DEBUG(downloader->pakfire, "cURL transfer done: %d - %s\n", code, curl_easy_strerror(code)); @@ -703,30 +974,98 @@ static int pakfire_transfer_done(struct pakfire_downloader* downloader, return 0; } -static size_t pakfire_downloader_write(char* data, size_t size, size_t nmemb, void* userdata) { - struct pakfire_transfer* transfer = (struct pakfire_transfer*)userdata; +static int pakfire_downloader_transfer_update(void* data, + curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { + struct pakfire_transfer* transfer = data; + + // Update the expected size + transfer->expected_size = dltotal; + + // Update the transferred counter + transfer->transferred = dlnow; + + // Do nothing if no progress indicator has been set up + if (!transfer->progress) + return 0; + + // Set maximum (because this might have changed) + pakfire_progress_set_max_value(transfer->progress, transfer->expected_size); + + // Update current value + return pakfire_progress_update(transfer->progress, transfer->transferred); +} + +static int pakfire_downloader_transfer_prepare_progress(struct pakfire_downloader* downloader, + struct pakfire_transfer* transfer, struct pakfire_progress* parent, int flags) { + const char* title = NULL; + int progress_flags = + PAKFIRE_PROGRESS_SHOW_PERCENTAGE | + PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED | + PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED | + PAKFIRE_PROGRESS_SHOW_ETA; int r; - // Do not write empty blocks - if (!nmemb) - return nmemb; + // If this has already been set up, we skip it + if (transfer->progress) + return 0; - // Update message digest - if (transfer->evp) { - r = EVP_DigestUpdate(transfer->evp, data, nmemb); - if (r != 1) { - ERROR(transfer->downloader->pakfire, "EVP_DigestUpdate failed: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - return 0; - } + // Show no progress if requested + if (flags & PAKFIRE_TRANSFER_NO_PROGRESS) + progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS; + + // Show no progress if we have a parent progress + else if (parent) + progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS; + + // Make a new progress meter + r = pakfire_progress_create(&transfer->progress, transfer->pakfire, progress_flags, parent); + if (r) + return r; + + // Set the title + title = pakfire_downloader_transfer_get_title(transfer); + if (title) { + r = pakfire_progress_set_title(transfer->progress, title); + if (r) + return r; } - // Write everything to the allocated file descriptor - return fwrite(data, size, nmemb, transfer->f); + // Configure callbacks + curl_easy_setopt(transfer->handle, CURLOPT_XFERINFOFUNCTION, + pakfire_downloader_transfer_update); + curl_easy_setopt(transfer->handle, CURLOPT_XFERINFODATA, transfer); + curl_easy_setopt(transfer->handle, CURLOPT_NOPROGRESS, 0L); + + // Start progress + r = pakfire_progress_start(transfer->progress, transfer->expected_size); + if (r) + return r; + + return 0; } -static int pakfire_downloader_prepare_transfer(struct pakfire_downloader* downloader, - struct pakfire_transfer* transfer) { +static int pakfire_downloader_transfer_prepare_tmpfile(struct pakfire_transfer* transfer) { + char path[PATH_MAX] = PAKFIRE_TMP_DIR "/pakfire-download.XXXXXX"; + FILE* f = NULL; + int r; + + // Allocate a temporary file + f = pakfire_mktemp(path, 0600); + if (!f) + return -errno; + + // Set this as output + r = pakfire_downloader_transfer_set_output(transfer, f); + if (r) + return r; + + // Store the path to the temporary file + return pakfire_string_set(transfer->tempfile, path); +} + +static int pakfire_downloader_transfer_prepare(struct pakfire_downloader* downloader, + struct pakfire_transfer* transfer, struct pakfire_progress* progress, int flags) { + char url[PATH_MAX]; int r; // Increment tries @@ -742,44 +1081,32 @@ static int pakfire_downloader_prepare_transfer(struct pakfire_downloader* downlo if (r) return r; - char* url = pakfire_url_join(transfer->mirror->url, transfer->url); - if (!url) { - ERROR(downloader->pakfire, "Error composing download URL: %m\n"); - return 1; - } + r = pakfire_url_join(url, transfer->mirror->url, transfer->url); + if (r) + return r; curl_easy_setopt(transfer->handle, CURLOPT_URL, url); - free(url); // Use baseurl } else if (*transfer->baseurl) { - char* url = pakfire_url_join(transfer->baseurl, transfer->url); - if (!url) { - ERROR(downloader->pakfire, "Error composing download URL: %m\n"); - return 1; - } + r = pakfire_url_join(url, transfer->baseurl, transfer->url); + if (r) + return r; curl_easy_setopt(transfer->handle, CURLOPT_URL, url); - free(url); - // Cannot compose URL + // Fail if we could not set the URL } else { - ERROR(downloader->pakfire, "Could not set URL\n"); - return 1; + ERROR(downloader->pakfire, "Invalid transfer %s\n", transfer->url); + return -EINVAL; } - // Open a temporary file to write the output to + // If we do not have an output file, we will create a temporary file if (!transfer->f) { - // Make path for the temporary file (must be on the same file system) - r = pakfire_string_format(transfer->tempfile, "%s.XXXXXX", transfer->path); - if (r) - return 1; - - transfer->f = pakfire_mktemp(transfer->tempfile, 0); - if (!transfer->f) { - ERROR(downloader->pakfire, "Could not create temporary file for download %s: %m\n", - transfer->tempfile); - return 1; + r = pakfire_downloader_transfer_prepare_tmpfile(transfer); + if (r) { + ERROR_ERRNO(transfer->pakfire, r, "Could not open a temporary file: %m\n"); + return r; } } @@ -806,46 +1133,8 @@ static int pakfire_downloader_prepare_transfer(struct pakfire_downloader* downlo } } - // Write all data to the callback function - curl_easy_setopt(transfer->handle, CURLOPT_WRITEFUNCTION, pakfire_downloader_write); - curl_easy_setopt(transfer->handle, CURLOPT_WRITEDATA, transfer); - - return 0; -} - -static int pakfire_downloader_update_progress(void* data, - curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { - struct pakfire_downloader* downloader = (struct pakfire_downloader*)data; - - // Set maximum (because this might have changed) - pakfire_progress_set_max_value(downloader->progress, dltotal); - - // Update current value - return pakfire_progress_update(downloader->progress, dlnow); -} - -static int pakfire_downloader_make_single_progress( - struct pakfire_downloader* downloader, struct pakfire_transfer* transfer) { - int r; - - // Reset the progress - pakfire_progress_reset(downloader->progress); - - // Set title - if (*transfer->title) { - r = pakfire_progress_set_title(downloader->progress, "%s", transfer->title); - if (r) - return r; - } - - // Update the progress - curl_easy_setopt(transfer->handle, CURLOPT_XFERINFOFUNCTION, - pakfire_downloader_update_progress); - curl_easy_setopt(transfer->handle, CURLOPT_XFERINFODATA, downloader); - curl_easy_setopt(transfer->handle, CURLOPT_NOPROGRESS, 0L); - - // Start the progress - r = pakfire_progress_start(downloader->progress, 0); + // Setup progress + r = pakfire_downloader_transfer_prepare_progress(downloader, transfer, progress, flags); if (r) return r; @@ -857,19 +1146,11 @@ static int pakfire_downloader_make_single_progress( */ static int pakfire_downloader_perform( struct pakfire_downloader* downloader, struct pakfire_transfer* transfer) { - const int has_progress = !(transfer->flags & PAKFIRE_TRANSFER_NOPROGRESS); int r; - // Create progress - if (has_progress) { - r = pakfire_downloader_make_single_progress(downloader, transfer); - if (r) - return r; - } - AGAIN: // Prepare the transfer - r = pakfire_downloader_prepare_transfer(downloader, transfer); + r = pakfire_downloader_transfer_prepare(downloader, transfer, NULL, 0); if (r) return r; @@ -883,11 +1164,32 @@ AGAIN: if (r == EAGAIN) goto AGAIN; - // Finish the progress - if (has_progress) { - r = pakfire_progress_finish(downloader->progress); - if (r) - return r; + return r; +} + +int pakfire_downloader_transfer_run(struct pakfire_transfer* transfer, int flags) { + struct pakfire_downloader* downloader = transfer->downloader; + int r; + +AGAIN: + // Prepare the transfer + r = pakfire_downloader_transfer_prepare(downloader, transfer, NULL, flags); + if (r) + return r; + + // Perform the action + r = curl_easy_perform(transfer->handle); + + // Handle the result + r = pakfire_transfer_done(downloader, transfer, r); + + // Repeat the transfer if asked + switch (-r) { + case EAGAIN: + goto AGAIN; + + default: + break; } return r; @@ -911,84 +1213,219 @@ int pakfire_downloader_retrieve( // Perform the download int r = pakfire_downloader_perform(downloader, transfer); - pakfire_transfer_free(transfer); + pakfire_downloader_transfer_free(transfer); return r; } -int pakfire_downloader_run(struct pakfire_downloader* downloader) { - struct pakfire_transfer* transfer; - unsigned int transfers = 0; +static size_t pakfire_downloader_total_downloadsize(struct pakfire_downloader* downloader) { + struct pakfire_transfer* transfer = NULL; + size_t size; + + size_t total_size = 0; + + TAILQ_FOREACH(transfer, &downloader->transfers_queued, nodes) { + size = pakfire_downloader_transfer_get_size(transfer); + + // Return zero if the size isn't known + if (!size) + return 0; + + total_size += size; + } + + return total_size; +} + +static unsigned int pakfire_downloader_total_queued_transfers(struct pakfire_downloader* downloader) { + struct pakfire_transfer* transfer = NULL; + unsigned int counter = 0; + + TAILQ_FOREACH(transfer, &downloader->transfers_queued, nodes) + counter++; + + return counter; +} + +static int pakfire_downloader_transfer_start(struct pakfire_downloader* downloader, + struct pakfire_transfer* transfer, struct pakfire_progress* progress) { + int r; + + DEBUG(downloader->pakfire, "Starting transfer %p...\n", transfer); + // Prepare the transfer + r = pakfire_downloader_transfer_prepare(downloader, transfer, progress, 0); + if (r) + return r; + + // Add the handle to cURL + r = curl_multi_add_handle(downloader->curl, transfer->handle); + if (r) { + ERROR(downloader->pakfire, "Adding handle failed: %s\n", curl_multi_strerror(r)); + return r; + } + + return r; +} + +static int pakfire_downloader_start_transfers(struct pakfire_downloader* downloader, + struct pakfire_progress* progress, unsigned int* running_transfers) { + struct pakfire_transfer* transfer = NULL; int r; + + // Keep running until we have reached our ceiling + while (*running_transfers < downloader->parallel) { + // We are done if there are no more transfers in the queue + if (TAILQ_EMPTY(&downloader->transfers_queued)) + break; + + // Fetch the next transfer + transfer = TAILQ_LAST(&downloader->transfers_queued, transfers_queued); + TAILQ_REMOVE(&downloader->transfers_queued, transfer, nodes); + + // Start the transfer + r = pakfire_downloader_transfer_start(downloader, transfer, progress); + if (r) { + TAILQ_INSERT_TAIL(&downloader->transfers_finished, transfer, nodes); + return r; + } + + TAILQ_INSERT_TAIL(&downloader->transfers_running, transfer, nodes); + (*running_transfers)++; + } + + return 0; +} + +int pakfire_downloader_run(struct pakfire_downloader* downloader, const char* title) { + struct pakfire_progress* progress = NULL; + struct pakfire_transfer* transfer = NULL; + unsigned int running_transfers = 0; + int progress_flags = + PAKFIRE_PROGRESS_SHOW_PERCENTAGE | + PAKFIRE_PROGRESS_SHOW_ETA; + int r = 0; + + CURLMsg* msg = NULL; int still_running; int msgs_left = -1; - int success = 0; + + // Fetch the total downloadsize + const size_t downloadsize = pakfire_downloader_total_downloadsize(downloader); + + // If we know the downloadsize, we can show more details + if (downloadsize) { + progress_flags |= + PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED | + PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED; + } else { + progress_flags |= + PAKFIRE_PROGRESS_SHOW_COUNTER; + } + + // Fetch the total number of queued transfers + const unsigned int num_queued_transfers = pakfire_downloader_total_queued_transfers(downloader); + + // Create a new progress indicator + r = pakfire_progress_create(&progress, downloader->pakfire, progress_flags, NULL); + if (r) + goto ERROR; + + // Set the title + if (title) { + r = pakfire_progress_set_title(progress, title); + if (r) + goto ERROR; + } + + // Start the progress + r = pakfire_progress_start(progress, (downloadsize) ? downloadsize : num_queued_transfers); + if (r) + goto ERROR; do { // Make sure that we have up to parallel transfers active - while (!TAILQ_EMPTY(&downloader->transfers) && transfers < downloader->parallel) { - // Fetch first item added - transfer = TAILQ_LAST(&downloader->transfers, transfers); - TAILQ_REMOVE(&downloader->transfers, transfer, nodes); - - // Prepare the transfer - r = pakfire_downloader_prepare_transfer(downloader, transfer); - if (r) { - pakfire_transfer_free(transfer); - return r; - } - - // Add the handle to cURL - r = curl_multi_add_handle(downloader->curl, transfer->handle); - if (r) { - ERROR(downloader->pakfire, "Adding handle failed\n"); - pakfire_transfer_free(transfer); - return r; - } + if (!r) { + r = pakfire_downloader_start_transfers(downloader, progress, &running_transfers); + if (r) + goto ERROR; + } - transfers++; + // Run cURL + r = curl_multi_perform(downloader->curl, &still_running); + if (r) { + ERROR(downloader->pakfire, "cURL error: %s\n", curl_easy_strerror(r)); + goto ERROR; } - curl_multi_perform(downloader->curl, &still_running); - - CURLMsg* msg; - while ((msg = curl_multi_info_read(downloader->curl, &msgs_left))) { - if (msg->msg == CURLMSG_DONE) { - transfers--; - - // Update reference to transfer - curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &transfer); - curl_multi_remove_handle(downloader->curl, transfer->handle); - - // Handle whatever comes after the transfer - r = pakfire_transfer_done(downloader, transfer, msg->data.result); - if (r) { - // We will try another mirror and therefore insert this - // transfer to be picked up again next - if (r == EAGAIN) { - TAILQ_INSERT_TAIL(&downloader->transfers, transfer, nodes); - continue; + for (;;) { + // Read the next message + msg = curl_multi_info_read(downloader->curl, &msgs_left); + if (!msg) + break; + + switch (msg->msg) { + case CURLMSG_DONE: + // Update reference to transfer + curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &transfer); + + // Remove the handle + curl_multi_remove_handle(downloader->curl, transfer->handle); + + // Remove the transfer from the running list + TAILQ_REMOVE(&downloader->transfers_running, transfer, nodes); + running_transfers--; + + // Call the done callback + r = pakfire_transfer_done(downloader, transfer, msg->data.result); + switch (-r) { + // If we are asked to try again we will re-queue the transfer + case EAGAIN: + TAILQ_INSERT_TAIL(&downloader->transfers_queued, transfer, nodes); + break; + + // Otherwise this transfer has finished + default: + TAILQ_INSERT_TAIL(&downloader->transfers_finished, transfer, nodes); + if (r) + goto ERROR; + break; } - success = r; - } + // Reset transfer + transfer = NULL; + break; - // Destroy transfer - pakfire_transfer_free(transfer); - // Log any other messages - } else { - ERROR(downloader->pakfire, "cURL reported error %d\n", msg->msg); + default: + ERROR(downloader->pakfire, "Received unhandled cURL message %d\n", msg->msg); + break; } } // Wait a little before going through the loop again if (still_running) curl_multi_wait(downloader->curl, NULL, 0, 100, NULL); - } while (still_running || !TAILQ_EMPTY(&downloader->transfers)); + } while (still_running || !TAILQ_EMPTY(&downloader->transfers_queued)); - // Success - return success; + // We are finished! + r = pakfire_progress_finish(progress); + if (r) + goto ERROR; + +ERROR: + // If the downloader was aborted, we need to fail all remaining running transfers + while (!TAILQ_EMPTY(&downloader->transfers_running)) { + transfer = TAILQ_LAST(&downloader->transfers_running, transfers_running); + TAILQ_REMOVE(&downloader->transfers_running, transfer, nodes); + + // Fail the transfer + pakfire_transfer_fail(downloader, transfer, 0); + } + + if (progress) + pakfire_progress_unref(progress); + + return r; } struct pakfire_mirrorlist { diff --git a/src/libpakfire/include/pakfire/downloader.h b/src/libpakfire/include/pakfire/downloader.h index f275355c8..64ddfb390 100644 --- a/src/libpakfire/include/pakfire/downloader.h +++ b/src/libpakfire/include/pakfire/downloader.h @@ -30,9 +30,8 @@ struct pakfire_mirrorlist; #include enum pakfire_transfer_flags { - PAKFIRE_TRANSFER_NONE = 0, - PAKFIRE_TRANSFER_NOPROGRESS = (1 << 0), - PAKFIRE_TRANSFER_NOTEMP = (1 << 1), + PAKFIRE_TRANSFER_NO_PROGRESS = (1 << 0), + PAKFIRE_TRANSFER_NOTEMP = (1 << 1), }; int pakfire_downloader_create(struct pakfire_downloader** downloader, struct pakfire* pakfire); @@ -40,6 +39,30 @@ int pakfire_downloader_create(struct pakfire_downloader** downloader, struct pak struct pakfire_downloader* pakfire_downloader_ref(struct pakfire_downloader* downloader); struct pakfire_downloader* pakfire_downloader_unref(struct pakfire_downloader* downloader); +struct pakfire_transfer; + +int pakfire_downloader_transfer_create( + struct pakfire_transfer** transfer, struct pakfire_downloader* downloader, const char* url); +struct pakfire_transfer* pakfire_downloader_transfer_ref(struct pakfire_transfer* transfer); +struct pakfire_transfer* pakfire_downloader_transfer_unref(struct pakfire_transfer* transfer); +const char* pakfire_downloader_transfer_get_title(struct pakfire_transfer* transfer); +int pakfire_downloader_transfer_set_title( + struct pakfire_transfer* transfer, const char* title); +int pakfire_downloader_transfer_set_baseurl( + struct pakfire_transfer* transfer, const char* baseurl); +int pakfire_downloader_transfer_set_mirrorlist( + struct pakfire_transfer* transfer, struct pakfire_mirrorlist* mirrors); +int pakfire_downloader_transfer_set_size( + struct pakfire_transfer* transfer, size_t size); +int pakfire_downloader_transfer_verify_digest( + struct pakfire_transfer* transfer, const enum pakfire_digest_types md, + const unsigned char* expected_digest, const size_t expected_digest_length); +int pakfire_downloader_transfer_set_output(struct pakfire_transfer* transfer, FILE* f); +int pakfire_downloader_transfer_set_target(struct pakfire_transfer* transfer, const char* path); + +int pakfire_downloader_transfer_run(struct pakfire_transfer* transfer, int flags); +int pakfire_downloader_run(struct pakfire_downloader* downloader, const char* title); + int pakfire_downloader_retrieve( struct pakfire_downloader* downloader, const char* baseurl, @@ -51,19 +74,7 @@ int pakfire_downloader_retrieve( const unsigned char* expected_digest, const size_t expected_digest_length, enum pakfire_transfer_flags flags); -int pakfire_downloader_add_transfer( - struct pakfire_downloader* downloader, - const char* baseurl, - struct pakfire_mirrorlist* mirrors, - const char* title, - const char* url, - const char* path, - enum pakfire_digest_types md, - const unsigned char* expected_digest, - const size_t expected_digest_length, - enum pakfire_transfer_flags flags); -int pakfire_downloader_run(struct pakfire_downloader* downloader); // Mirror lists diff --git a/src/libpakfire/include/pakfire/repo.h b/src/libpakfire/include/pakfire/repo.h index 9afb4bffd..82c7b10ed 100644 --- a/src/libpakfire/include/pakfire/repo.h +++ b/src/libpakfire/include/pakfire/repo.h @@ -123,6 +123,12 @@ Id pakfire_repo_add_solvable(struct pakfire_repo* repo); int pakfire_repo_add_archive(struct pakfire_repo* repo, struct pakfire_archive* archive, struct pakfire_package** package); +int pakfire_repo_download_package( + struct pakfire_repo* repo, + struct pakfire_downloader* downloader, + struct pakfire_package* pkg, + struct pakfire_transfer** transfer); + int pakfire_repo_add(struct pakfire_repo* repo, const char* path, struct pakfire_package** package); diff --git a/src/libpakfire/repo.c b/src/libpakfire/repo.c index ab70091aa..bc8833e6e 100644 --- a/src/libpakfire/repo.c +++ b/src/libpakfire/repo.c @@ -63,6 +63,7 @@ struct pakfire_repo_appdata { // Base URL char baseurl[PATH_MAX]; + char expanded_baseurl[PATH_MAX]; // Refresh Interval time_t refresh; @@ -90,6 +91,8 @@ struct pakfire_repo { struct pakfire_key* key; }; +static const char* pakfire_repo_get_expanded_baseurl(struct pakfire_repo* repo); + int pakfire_repo_is_internal(struct pakfire_repo* repo) { const char* name = pakfire_repo_get_name(repo); if (!name) @@ -214,50 +217,51 @@ static int __pakfire_repo_path(struct pakfire_repo* repo, return __pakfire_cache_path(repo->pakfire, path, length, "%s/%s", name, buffer); } -static int pakfire_repo_retrieve( - struct pakfire_repo* repo, - const char* title, - const char* url, - const char* path, - enum pakfire_digest_types md, - const unsigned char* expected_digest, - const size_t expected_digest_length, - enum pakfire_transfer_flags flags) { - struct pakfire_downloader* downloader = NULL; - char* baseurl = NULL; +/* + This is a convenience function to create a transfer with the + settings of this repository. +*/ +static int pakfire_repo_create_transfer(struct pakfire_transfer** transfer, + struct pakfire_repo* repo, struct pakfire_downloader* downloader, const char* url) { + struct pakfire_mirrorlist* mirrorlist = NULL; + struct pakfire_transfer* t = NULL; + const char* baseurl = NULL; int r; - // Create downloader - r = pakfire_downloader_create(&downloader, repo->pakfire); + // Create a new transfer + r = pakfire_downloader_transfer_create(&t, downloader, url); if (r) goto ERROR; - // Prepare the baseurl - baseurl = pakfire_repo_url_replace(repo, repo->appdata->baseurl); - if (!baseurl) { - r = -errno; - goto ERROR; + // Set the baseurl + baseurl = pakfire_repo_get_expanded_baseurl(repo); + if (baseurl) { + r = pakfire_downloader_transfer_set_baseurl(t, baseurl); + if (r) + goto ERROR; } -#if 0 - // Fetch mirrorlist - struct pakfire_mirrorlist* mirrorlist = pakfire_repo_get_mirrorlist(repo); -#endif + // Set the mirrorlist + mirrorlist = pakfire_repo_get_mirrorlist(repo); + if (mirrorlist) { + r = pakfire_downloader_transfer_set_mirrorlist(t, mirrorlist); + if (r) + goto ERROR; + } - // Retrieve the database file - r = pakfire_downloader_retrieve(downloader, baseurl, NULL, - title, url, path, md, expected_digest, expected_digest_length, flags); + // Success + *transfer = t; + r = 0; -#if 0 - if (mirrorlist) - pakfire_mirrorlist_unref(mirrorlist); -#endif + goto CLEANUP; ERROR: - if (downloader) - pakfire_downloader_unref(downloader); - if (baseurl) - free(baseurl); + if (t) + pakfire_downloader_transfer_unref(t); + +CLEANUP: + if (mirrorlist) + pakfire_mirrorlist_unref(mirrorlist); return r; } @@ -423,6 +427,8 @@ struct pakfire_mirrorlist* pakfire_repo_get_mirrorlist(struct pakfire_repo* repo static int pakfire_repo_download_database(struct pakfire_repo* repo, const char* filename, const char* path) { + struct pakfire_downloader* downloader = NULL; + struct pakfire_transfer* transfer = NULL; char title[NAME_MAX]; char url[PATH_MAX]; int r; @@ -445,7 +451,38 @@ static int pakfire_repo_download_database(struct pakfire_repo* repo, if (r) return r; - return pakfire_repo_retrieve(repo, title, url, path, 0, NULL, 0, 0); + // Create a downloader + r = pakfire_downloader_create(&downloader, repo->pakfire); + if (r) + goto ERROR; + + // Create a new transfer + r = pakfire_repo_create_transfer(&transfer, repo, downloader, url); + if (r) + goto ERROR; + + // Set title + r = pakfire_downloader_transfer_set_title(transfer, title); + if (r) + goto ERROR; + + // Set path + r = pakfire_downloader_transfer_set_target(transfer, path); + if (r) + goto ERROR; + + // Run the transfer + r = pakfire_downloader_transfer_run(transfer, 0); + if (r) + goto ERROR; + +ERROR: + if (transfer) + pakfire_downloader_transfer_unref(transfer); + if (downloader) + pakfire_downloader_unref(downloader); + + return r; } static int pakfire_repo_read_database(struct pakfire_repo* repo, const char* path) { @@ -540,6 +577,8 @@ ERROR: } static int pakfire_repo_refresh_mirrorlist(struct pakfire_repo* repo, const int force) { + struct pakfire_downloader* downloader = NULL; + struct pakfire_transfer* transfer = NULL; char* url = NULL; int r; @@ -579,10 +618,26 @@ static int pakfire_repo_refresh_mirrorlist(struct pakfire_repo* repo, const int goto ERROR; } - r = pakfire_repo_retrieve(repo, NULL, url, repo->appdata->mirrorlist, - 0, NULL, 0, PAKFIRE_TRANSFER_NOPROGRESS); + // Create a new downloader + r = pakfire_downloader_create(&downloader, repo->pakfire); + if (r) + goto ERROR; + + // Create a new transfer + r = pakfire_repo_create_transfer(&transfer, repo, downloader, url); + if (r) + goto ERROR; + + // Run the transfer + r = pakfire_downloader_transfer_run(transfer, PAKFIRE_TRANSFER_NO_PROGRESS); + if (r) + goto ERROR; ERROR: + if (transfer) + pakfire_downloader_transfer_unref(transfer); + if (downloader) + pakfire_downloader_unref(downloader); if (url) free(url); @@ -590,6 +645,10 @@ ERROR: } static int pakfire_repo_download_metadata(struct pakfire_repo* repo, const char* path, int force) { + struct pakfire_downloader* downloader = NULL; + struct pakfire_transfer* transfer = NULL; + int r; + // Local repositories don't need to download metadata if (pakfire_repo_is_local(repo)) return 0; @@ -619,8 +678,33 @@ static int pakfire_repo_download_metadata(struct pakfire_repo* repo, const char* } } - return pakfire_repo_retrieve(repo, NULL, "repodata/repomd.json", path, - 0, NULL, 0, PAKFIRE_TRANSFER_NOPROGRESS); + // Create a downloader + r = pakfire_downloader_create(&downloader, repo->pakfire); + if (r) + goto ERROR; + + // Create a new transfer + r = pakfire_repo_create_transfer(&transfer, repo, downloader, "repodata/repomd.json"); + if (r) + goto ERROR; + + // Set the target path + r = pakfire_downloader_transfer_set_target(transfer, path); + if (r) + goto ERROR; + + // Perform the transfer + r = pakfire_downloader_transfer_run(transfer, PAKFIRE_TRANSFER_NO_PROGRESS); + if (r) + goto ERROR; + +ERROR: + if (transfer) + pakfire_downloader_transfer_unref(transfer); + if (downloader) + pakfire_downloader_unref(downloader); + + return r; } static int pakfire_repo_refresh_metadata(struct pakfire_repo* repo, const int force) { @@ -879,6 +963,32 @@ PAKFIRE_EXPORT const char* pakfire_repo_get_baseurl(struct pakfire_repo* repo) { return repo->appdata->baseurl; } +static const char* pakfire_repo_get_expanded_baseurl(struct pakfire_repo* repo) { + char* url = NULL; + int r; + + // Return NULL if baseurl isn't set + if (!*repo->appdata->baseurl) + return NULL; + + if (!*repo->appdata->expanded_baseurl) { + url = pakfire_repo_url_replace(repo, repo->appdata->baseurl); + if (!url) + goto ERROR; + + // Store the result + r = pakfire_string_set(repo->appdata->expanded_baseurl, url); + if (r) + goto ERROR; + } + +ERROR: + if (url) + free(url); + + return (*repo->appdata->expanded_baseurl) ? repo->appdata->expanded_baseurl : NULL; +} + PAKFIRE_EXPORT int pakfire_repo_set_baseurl(struct pakfire_repo* repo, const char* baseurl) { int r; @@ -887,6 +997,9 @@ PAKFIRE_EXPORT int pakfire_repo_set_baseurl(struct pakfire_repo* repo, const cha if (r) return r; + // Invalidate the expanded URL + *repo->appdata->expanded_baseurl = '\0'; + // Update sub-priority pakfire_repo_update_subpriority(repo); @@ -1055,9 +1168,90 @@ PAKFIRE_EXPORT int pakfire_repo_is_installed_repo(struct pakfire_repo* repo) { return (r == 0); } +int pakfire_repo_download_package(struct pakfire_repo* repo, struct pakfire_downloader* downloader, + struct pakfire_package* pkg, struct pakfire_transfer** transfer) { + struct pakfire_transfer* t = NULL; + const unsigned char* digest = NULL; + const char* cache_path = NULL; + const char* nevra = NULL; + const char* url = NULL; + int r = 1; + + // Fetch nevra + nevra = pakfire_package_get_string(pkg, PAKFIRE_PKG_NEVRA); + if (!nevra) + goto ERROR; + + // Where do we store the package? + cache_path = pakfire_package_get_string(pkg, PAKFIRE_PKG_CACHE_PATH); + if (!cache_path) { + ERROR(repo->pakfire, "Could not retrieve package cache path for %s: %m\n", nevra); + goto ERROR; + } + + // Where do we find the package on the server? + url = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH); + if (!url) { + ERROR(repo->pakfire, "Could not retrieve package path for %s: %m\n", nevra); + goto ERROR; + } + + enum pakfire_digest_types digest_type = 0; + size_t digest_length = 0; + + // Retrieve package digest + digest = pakfire_package_get_digest(pkg, &digest_type, &digest_length); + if (!digest) { + ERROR(repo->pakfire, "Package %s has no digest set: %m\n", nevra); + goto ERROR; + } + + // Create a new transfer + r = pakfire_repo_create_transfer(&t, repo, downloader, url); + if (r) + goto ERROR; + + // Set title + r = pakfire_downloader_transfer_set_title(t, nevra); + if (r) + goto ERROR; + + // Fetch downloadsize + size_t downloadsize = pakfire_package_get_num(pkg, PAKFIRE_PKG_DOWNLOADSIZE, 0); + + // Set size + if (downloadsize > 0) { + r = pakfire_downloader_transfer_set_size(t, downloadsize); + if (r) + goto ERROR; + } + + // Set target + r = pakfire_downloader_transfer_set_target(t, cache_path); + if (r) + goto ERROR; + + // Set digest + r = pakfire_downloader_transfer_verify_digest(t, digest_type, digest, digest_length); + if (r) + goto ERROR; + + // Success + *transfer = t; + + return 0; + +ERROR: + if (t) + pakfire_downloader_transfer_unref(t); + + return r; +} + static int pakfire_repo_download(struct pakfire_repo* repo, const char* url, struct pakfire_package** package) { struct pakfire_downloader* downloader = NULL; + struct pakfire_transfer* transfer = NULL; FILE* f = NULL; int r; @@ -1075,9 +1269,18 @@ static int pakfire_repo_download(struct pakfire_repo* repo, const char* url, if (r) goto ERROR; - // Download the file - r = pakfire_downloader_retrieve(downloader, NULL, NULL, NULL, - url, path, 0, NULL, 0, PAKFIRE_TRANSFER_NOTEMP); + // Create a new transfer + r = pakfire_downloader_transfer_create(&transfer, downloader, url); + if (r) + goto ERROR; + + // Set destination + r = pakfire_downloader_transfer_set_output(transfer, f); + if (r) + goto ERROR; + + // Run the download + r = pakfire_downloader_transfer_run(transfer, 0); if (r) goto ERROR; @@ -1087,6 +1290,8 @@ static int pakfire_repo_download(struct pakfire_repo* repo, const char* url, goto ERROR; ERROR: + if (transfer) + pakfire_downloader_transfer_unref(transfer); if (downloader) pakfire_downloader_unref(downloader); if (f) @@ -1107,8 +1312,13 @@ int pakfire_repo_add(struct pakfire_repo* repo, const char* path, } // Download the package if we got given a URL - if (pakfire_string_is_url(path)) - return pakfire_repo_download(repo, path, package); + if (pakfire_string_is_url(path)) { + r = pakfire_repo_download(repo, path, package); + if (r) + ERROR(repo->pakfire, "Could not download %s\n", path); + + return r; + } // Try to open the archive r = pakfire_archive_open(&archive, repo->pakfire, path); diff --git a/src/libpakfire/transaction.c b/src/libpakfire/transaction.c index 4dbfdc397..da7395fbc 100644 --- a/src/libpakfire/transaction.c +++ b/src/libpakfire/transaction.c @@ -1808,68 +1808,25 @@ ERROR: static int pakfire_transaction_download_package(struct pakfire_transaction* transaction, struct pakfire_downloader* downloader, struct pakfire_package* pkg) { - int r = 1; struct pakfire_repo* repo = NULL; - struct pakfire_mirrorlist* mirrorlist = NULL; - char* __baseurl = NULL; + struct pakfire_transfer* transfer = NULL; + int r = 1; // Fetch the repository to download from repo = pakfire_package_get_repo(pkg); if (!repo) goto ERROR; - // Fetch baseurl - const char* baseurl = pakfire_repo_get_baseurl(repo); - - // Prepare the base url - __baseurl = pakfire_repo_url_replace(repo, baseurl); - if (!__baseurl) { - r = -errno; - goto ERROR; - } - - // Fetch mirrorlist - mirrorlist = pakfire_repo_get_mirrorlist(repo); - - const char* nevra = pakfire_package_get_string(pkg, PAKFIRE_PKG_NEVRA); - if (!nevra) - goto ERROR; - - // Where do we find the package on the server? - const char* path = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH); - if (!path) { - ERROR(transaction->pakfire, "Could not retrieve package path for %s: %m\n", nevra); - goto ERROR; - } - - // Where do we store the package? - const char* cache_path = pakfire_package_get_string(pkg, PAKFIRE_PKG_CACHE_PATH); - if (!cache_path) { - ERROR(transaction->pakfire, "Could not retrieve package cache path for %s: %m\n", nevra); - goto ERROR; - } - - enum pakfire_digest_types digest_type = 0; - size_t digest_length = 0; - - // Retrieve package digest - const unsigned char* digest = pakfire_package_get_digest(pkg, &digest_type, &digest_length); - if (!digest) { - ERROR(transaction->pakfire, "Package %s has no digest set: %m\n", nevra); + // Create a transfer for this package + r = pakfire_repo_download_package(repo, downloader, pkg, &transfer); + if (r) goto ERROR; - } - - // Add transfer to downloader - r = pakfire_downloader_add_transfer(downloader, __baseurl, mirrorlist, - nevra, path, cache_path, digest_type, digest, digest_length, 0); ERROR: - if (mirrorlist) - pakfire_mirrorlist_unref(mirrorlist); + if (transfer) + pakfire_downloader_transfer_unref(transfer); if (repo) pakfire_repo_unref(repo); - if (__baseurl) - free(__baseurl); return r; } @@ -1951,7 +1908,7 @@ PAKFIRE_EXPORT int pakfire_transaction_download(struct pakfire_transaction* tran } // Run the downloader - r = pakfire_downloader_run(downloader); + r = pakfire_downloader_run(downloader, _("Downloading Packages")); ERROR: pakfire_downloader_unref(downloader);