#############################################################################*/
#include <errno.h>
+#include <fcntl.h>
#include <stdlib.h>
#include <sys/queue.h>
#include <unistd.h>
};
struct pakfire_transfer {
+ struct pakfire* pakfire;
+ int nrefs;
+
+ // Reference to the downloader
struct pakfire_downloader* downloader;
+ // Reference to the progress indicator
+ struct pakfire_progress* progress;
+
TAILQ_ENTRY(pakfire_transfer) nodes;
+
+ // cURL handle
CURL* handle;
char title[NAME_MAX];
enum pakfire_transfer_flags flags;
int tries;
+ // Size
+ size_t expected_size;
+ size_t transferred;
+
// Temporary file
char tempfile[PATH_MAX];
FILE* f;
struct pakfire* pakfire;
int nrefs;
- struct pakfire_progress* progress;
+// struct pakfire_progress* progress;
unsigned int parallel;
// cURL multi handle
CURLM* curl;
- TAILQ_HEAD(transfers, pakfire_transfer) transfers;
-};
-static char* pakfire_url_join(const char* part1, const char* part2) {
- char* url = NULL;
+ // Transfers
+ TAILQ_HEAD(transfers_queued, pakfire_transfer) transfers_queued;
+ TAILQ_HEAD(transfers_running, pakfire_transfer) transfers_running;
+ TAILQ_HEAD(transfers_finished, pakfire_transfer) transfers_finished;
+};
- int r = asprintf(&url, "%s/%s", part1, part2);
- if (r < 0)
- return NULL;
+#define pakfire_url_join(s, part1, part2) \
+ __pakfire_url_join(s, sizeof(s), part1, part2)
- return url;
+static int __pakfire_url_join(char* s, const size_t length, const char* part1, const char* part2) {
+ return __pakfire_string_format(s, length, "%s/%s", part1, part2);
}
static int pakfire_downloader_setup_curl(struct pakfire_downloader* downloader) {
return 0;
}
-static void pakfire_transfer_free(struct pakfire_transfer* transfer) {
+static void pakfire_downloader_transfer_free(struct pakfire_transfer* transfer) {
if (transfer->handle)
curl_easy_cleanup(transfer->handle);
if (transfer->mirrors)
pakfire_mirrorlist_unref(transfer->mirrors);
+ if (transfer->progress)
+ pakfire_progress_unref(transfer->progress);
+ if (transfer->pakfire)
+ pakfire_unref(transfer->pakfire);
free(transfer);
}
static void pakfire_downloader_free(struct pakfire_downloader* downloader) {
- // Free any unprocessed transfers
- struct pakfire_transfer* transfer;
- while (!TAILQ_EMPTY(&downloader->transfers)) {
- transfer = TAILQ_LAST(&downloader->transfers, transfers);
- TAILQ_REMOVE(&downloader->transfers, transfer, nodes);
- pakfire_transfer_free(transfer);
+ struct pakfire_transfer* transfer = NULL;
+
+ // Free any queued transfers
+ while (!TAILQ_EMPTY(&downloader->transfers_queued)) {
+ transfer = TAILQ_LAST(&downloader->transfers_queued, transfers_queued);
+ TAILQ_REMOVE(&downloader->transfers_queued, transfer, nodes);
+
+ pakfire_downloader_transfer_unref(transfer);
}
- if (downloader->progress)
- pakfire_progress_unref(downloader->progress);
+ // Free any running transfers
+ while (!TAILQ_EMPTY(&downloader->transfers_running)) {
+ transfer = TAILQ_LAST(&downloader->transfers_running, transfers_running);
+ TAILQ_REMOVE(&downloader->transfers_running, transfer, nodes);
+
+ pakfire_downloader_transfer_unref(transfer);
+ }
+
+ // Free any finished transfers
+ while (!TAILQ_EMPTY(&downloader->transfers_finished)) {
+ transfer = TAILQ_LAST(&downloader->transfers_finished, transfers_finished);
+ TAILQ_REMOVE(&downloader->transfers_finished, transfer, nodes);
+
+ pakfire_downloader_transfer_unref(transfer);
+ }
if (downloader->curl)
curl_multi_cleanup(downloader->curl);
}
int pakfire_downloader_create(struct pakfire_downloader** downloader, struct pakfire* pakfire) {
+ struct pakfire_downloader* d = NULL;
int r;
// Fail if pakfire is running in offline mode
if (pakfire_has_flag(pakfire, PAKFIRE_FLAGS_OFFLINE)) {
ERROR(pakfire, "Cannot initialize downloader in offline mode\n");
- errno = ENOTSUP;
- return 1;
+ return -ENOTSUP;
}
- struct pakfire_downloader* d = calloc(1, sizeof(*d));
+ // Allocate a new object
+ d = calloc(1, sizeof(*d));
if (!d)
- return ENOMEM;
+ return -ENOMEM;
// Store reference to Pakfire
d->pakfire = pakfire_ref(pakfire);
// Set parallelism
d->parallel = MAX_PARALLEL;
- // Init transfers queue
- TAILQ_INIT(&d->transfers);
+ // Init transfer queues
+ TAILQ_INIT(&d->transfers_queued);
+ TAILQ_INIT(&d->transfers_running);
+ TAILQ_INIT(&d->transfers_finished);
// Setup cURL
r = pakfire_downloader_setup_curl(d);
if (r)
goto ERROR;
- // Create the progress indicator
- r = pakfire_progress_create(&d->progress, d->pakfire,
- PAKFIRE_PROGRESS_SHOW_PERCENTAGE |
- PAKFIRE_PROGRESS_SHOW_ETA |
- PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED |
- PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED);
- if (r)
- goto ERROR;
-
+ // Success
*downloader = d;
return 0;
}
#endif
+static size_t pakfire_downloader_transfer_write(
+ char* data, size_t size, size_t nmemb, void* p) {
+ struct pakfire_transfer* transfer = p;
+ int r;
+
+ // Do not write empty blocks
+ if (!nmemb)
+ return nmemb;
+
+ // Update message digest
+ if (transfer->evp) {
+ r = EVP_DigestUpdate(transfer->evp, data, nmemb);
+ if (r != 1) {
+ ERROR(transfer->downloader->pakfire, "EVP_DigestUpdate failed: %s\n",
+ ERR_error_string(ERR_get_error(), NULL));
+ return 0;
+ }
+ }
+
+ // Write everything to the allocated file descriptor
+ return fwrite(data, size, nmemb, transfer->f);
+}
+
+static int pakfire_downloader_transfer_setup(
+ struct pakfire_downloader* downloader, struct pakfire_transfer* transfer) {
+ struct pakfire_config* config = NULL;
+ const char* proxy = NULL;
+ int r;
+
+ // Fetch global configuration
+ config = pakfire_get_config(downloader->pakfire);
+
+ // Set global configuration
+ if (config) {
+ proxy = pakfire_config_get(config, "general", "proxy", NULL);
+ if (proxy)
+ curl_easy_setopt(transfer->handle, CURLOPT_PROXY, proxy);
+ }
+
+ // Be a good net citizen and set a user agent
+ curl_easy_setopt(transfer->handle, CURLOPT_USERAGENT, PACKAGE_NAME "/" PACKAGE_VERSION);
+
+#ifdef ENABLE_DEBUG
+ // Enable logging/debugging
+ curl_easy_setopt(transfer->handle, CURLOPT_VERBOSE, 1L);
+
+ curl_easy_setopt(transfer->handle, CURLOPT_DEBUGFUNCTION, debug_callback);
+ curl_easy_setopt(transfer->handle, CURLOPT_DEBUGDATA, downloader->pakfire);
+#endif
+
+ // Limit protocols to HTTPS, HTTP, FTP and FILE
+ curl_easy_setopt(transfer->handle, CURLOPT_PROTOCOLS_STR, "HTTPS,HTTP,FTP,FILE");
+
+ // Reference back to this transfer
+ curl_easy_setopt(transfer->handle, CURLOPT_PRIVATE, transfer);
+
+ // Follow any redirects
+ curl_easy_setopt(transfer->handle, CURLOPT_FOLLOWLOCATION, 1);
+
+ // Write all data to the callback function
+ curl_easy_setopt(transfer->handle,
+ CURLOPT_WRITEFUNCTION, pakfire_downloader_transfer_write);
+ curl_easy_setopt(transfer->handle, CURLOPT_WRITEDATA, transfer);
+
+ // Success
+ r = 0;
+
+ // Cleanup
+ if (config)
+ pakfire_config_unref(config);
+
+ return r;
+}
+
+int pakfire_downloader_transfer_create(struct pakfire_transfer** transfer,
+ struct pakfire_downloader* downloader, const char* url) {
+ struct pakfire_transfer* t = NULL;
+ int r;
+
+ // Allocate a new transfer
+ t = calloc(1, sizeof(*t));
+ if (!t)
+ return -errno;
+
+ // Store a reference to Pakfire
+ t->pakfire = pakfire_ref(downloader->pakfire);
+
+ // Initialize the reference counter
+ t->nrefs = 1;
+
+ // Store a reference to the downloader
+ t->downloader = pakfire_downloader_ref(downloader);
+
+ // Store the URL
+ r = pakfire_string_set(t->url, url);
+ if (r)
+ goto ERROR;
+
+ // Allocate a handle
+ t->handle = curl_easy_init();
+ if (!t->handle) {
+ r = 1;
+ goto ERROR;
+ }
+
+ // Setup the transfer
+ r = pakfire_downloader_transfer_setup(downloader, t);
+ if (r)
+ goto ERROR;
+
+ // Push this transfer onto the queue
+ TAILQ_INSERT_HEAD(&downloader->transfers_queued,
+ pakfire_downloader_transfer_ref(t), nodes);
+
+ // Return the reference
+ *transfer = t;
+
+ return 0;
+
+ERROR:
+ if (t)
+ pakfire_downloader_transfer_unref(t);
+
+ return r;
+}
+
+struct pakfire_transfer* pakfire_downloader_transfer_ref(struct pakfire_transfer* transfer) {
+ ++transfer->nrefs;
+
+ return transfer;
+}
+
+struct pakfire_transfer* pakfire_downloader_transfer_unref(struct pakfire_transfer* transfer) {
+ if (--transfer->nrefs > 0)
+ return transfer;
+
+ pakfire_downloader_transfer_free(transfer);
+ return NULL;
+}
+
+const char* pakfire_downloader_transfer_get_title(struct pakfire_transfer* transfer) {
+ char path[PATH_MAX];
+ int r;
+
+ // Default to the filename if no title is set
+ if (!*transfer->title) {
+ r = pakfire_basename(path, transfer->url);
+ if (r)
+ return NULL;
+
+ // Store the title
+ r = pakfire_downloader_transfer_set_title(transfer, path);
+ if (r)
+ return NULL;
+ }
+
+ return transfer->title;
+}
+
+int pakfire_downloader_transfer_set_title(
+ struct pakfire_transfer* transfer, const char* title) {
+ return pakfire_string_set(transfer->title, title);
+}
+
+int pakfire_downloader_transfer_set_baseurl(
+ struct pakfire_transfer* transfer, const char* baseurl) {
+ return pakfire_string_set(transfer->baseurl, baseurl);
+}
+
+int pakfire_downloader_transfer_set_mirrorlist(
+ struct pakfire_transfer* transfer, struct pakfire_mirrorlist* mirrors) {
+ if (transfer->mirrors)
+ pakfire_mirrorlist_unref(transfer->mirrors);
+
+ transfer->mirrors = pakfire_mirrorlist_ref(mirrors);
+
+ return 0;
+}
+
+static size_t pakfire_downloader_transfer_get_size(struct pakfire_transfer* transfer) {
+ return transfer->expected_size;
+}
+
+int pakfire_downloader_transfer_set_size(
+ struct pakfire_transfer* transfer, size_t size) {
+ transfer->expected_size = size;
+
+ return 0;
+}
+
+int pakfire_downloader_transfer_verify_digest(
+ struct pakfire_transfer* transfer, const enum pakfire_digest_types md,
+ const unsigned char* expected_digest, const size_t expected_digest_length) {
+ // Check inputs
+ if (!expected_digest || !expected_digest_length)
+ return -EINVAL;
+
+ // Expected digest length cannot be too long
+ if (expected_digest_length > sizeof(transfer->expected_digest))
+ return -ENOBUFS;
+
+ // Store digest type
+ switch (md) {
+ case PAKFIRE_DIGEST_SHA3_512:
+ transfer->md = EVP_sha3_512();
+ break;
+
+ case PAKFIRE_DIGEST_SHA3_256:
+ transfer->md = EVP_sha3_256();
+ break;
+
+ case PAKFIRE_DIGEST_BLAKE2B512:
+ transfer->md = EVP_blake2b512();
+ break;
+
+ case PAKFIRE_DIGEST_BLAKE2S256:
+ transfer->md = EVP_blake2s256();
+ break;
+
+ case PAKFIRE_DIGEST_SHA2_512:
+ transfer->md = EVP_sha512();
+ break;
+
+ case PAKFIRE_DIGEST_SHA2_256:
+ transfer->md = EVP_sha256();
+ break;
+
+ default:
+ return -ENOTSUP;
+ }
+
+ // Store the expected digest and its length
+ memcpy(transfer->expected_digest, expected_digest, expected_digest_length);
+ transfer->expected_digest_length = expected_digest_length;
+
+ return 0;
+}
+
+int pakfire_downloader_transfer_set_output(struct pakfire_transfer* transfer, FILE* f) {
+ transfer->f = f;
+
+ return 0;
+}
+
+int pakfire_downloader_transfer_set_target(
+ struct pakfire_transfer* transfer, const char* path) {
+ return pakfire_string_set(transfer->path, path);
+}
+
static struct pakfire_transfer* pakfire_downloader_create_transfer(
struct pakfire_downloader* downloader,
const char* baseurl,
return transfer;
ERROR:
- pakfire_transfer_free(transfer);
+ pakfire_downloader_transfer_free(transfer);
return NULL;
}
-int pakfire_downloader_add_transfer(
- struct pakfire_downloader* downloader,
- const char* baseurl,
- struct pakfire_mirrorlist* mirrors,
- const char* title,
- const char* url,
- const char* path,
- enum pakfire_digest_types md,
- const unsigned char* expected_digest,
- const size_t expected_digest_length,
- enum pakfire_transfer_flags flags) {
- struct pakfire_transfer* transfer = pakfire_downloader_create_transfer(
- downloader, baseurl, mirrors, title, url, path, md, expected_digest, expected_digest_length, flags);
- if (!transfer)
- return 1;
-
- // Push this transfer onto the queue
- TAILQ_INSERT_HEAD(&downloader->transfers, transfer, nodes);
-
- return 0;
-}
-
static int pakfire_transfer_select_mirror(struct pakfire_downloader* downloader,
struct pakfire_transfer* transfer) {
// Choose the next mirror
struct utimbuf times;
int r;
+ // Nothing to do if path isn't set
+ if (!*transfer->path)
+ return 0;
+
DEBUG(downloader->pakfire,
"Download successful. Storing result in %s\n", transfer->path);
- if (!(transfer->flags & PAKFIRE_TRANSFER_NOTEMP)) {
- // Remove destination (if it exists)
- unlink(transfer->path);
+ // Flush any buffered data out to disk
+ fflush(transfer->f);
- // Move the temporary file to its destination
- r = link(transfer->tempfile, transfer->path);
- if (r) {
- ERROR(downloader->pakfire, "Could not link destination file %s: %m\n",
- transfer->path);
- return r;
- }
+ // Make sure the parent directory exists
+ r = pakfire_mkparentdir(transfer->path, 0755);
+ if (r)
+ return r;
+
+ // Unlink the destination file (if it exists)
+ unlink(transfer->path);
+
+ // Move the temporary file to its destination
+ r = link(transfer->tempfile, transfer->path);
+ if (r) {
+ ERROR(downloader->pakfire, "Could not link destination file %s: %m\n",
+ transfer->path);
+ return r;
}
// Filetime
curl_off_t download_size;
curl_off_t speed;
+ // Finish progress
+ r = pakfire_progress_finish(transfer->progress);
+ if (r)
+ return r;
+
DEBUG(downloader->pakfire, "cURL transfer done: %d - %s\n",
code, curl_easy_strerror(code));
return 0;
}
-static size_t pakfire_downloader_write(char* data, size_t size, size_t nmemb, void* userdata) {
- struct pakfire_transfer* transfer = (struct pakfire_transfer*)userdata;
+static int pakfire_downloader_transfer_update(void* data,
+ curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
+ struct pakfire_transfer* transfer = data;
+
+ // Update the expected size
+ transfer->expected_size = dltotal;
+
+ // Update the transferred counter
+ transfer->transferred = dlnow;
+
+ // Do nothing if no progress indicator has been set up
+ if (!transfer->progress)
+ return 0;
+
+ // Set maximum (because this might have changed)
+ pakfire_progress_set_max_value(transfer->progress, transfer->expected_size);
+
+ // Update current value
+ return pakfire_progress_update(transfer->progress, transfer->transferred);
+}
+
+static int pakfire_downloader_transfer_prepare_progress(struct pakfire_downloader* downloader,
+ struct pakfire_transfer* transfer, struct pakfire_progress* parent, int flags) {
+ const char* title = NULL;
+ int progress_flags =
+ PAKFIRE_PROGRESS_SHOW_PERCENTAGE |
+ PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED |
+ PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED |
+ PAKFIRE_PROGRESS_SHOW_ETA;
int r;
- // Do not write empty blocks
- if (!nmemb)
- return nmemb;
+ // If this has already been set up, we skip it
+ if (transfer->progress)
+ return 0;
- // Update message digest
- if (transfer->evp) {
- r = EVP_DigestUpdate(transfer->evp, data, nmemb);
- if (r != 1) {
- ERROR(transfer->downloader->pakfire, "EVP_DigestUpdate failed: %s\n",
- ERR_error_string(ERR_get_error(), NULL));
- return 0;
- }
+ // Show no progress if requested
+ if (flags & PAKFIRE_TRANSFER_NO_PROGRESS)
+ progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
+
+ // Show no progress if we have a parent progress
+ else if (parent)
+ progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS;
+
+ // Make a new progress meter
+ r = pakfire_progress_create(&transfer->progress, transfer->pakfire, progress_flags, parent);
+ if (r)
+ return r;
+
+ // Set the title
+ title = pakfire_downloader_transfer_get_title(transfer);
+ if (title) {
+ r = pakfire_progress_set_title(transfer->progress, title);
+ if (r)
+ return r;
}
- // Write everything to the allocated file descriptor
- return fwrite(data, size, nmemb, transfer->f);
+ // Configure callbacks
+ curl_easy_setopt(transfer->handle, CURLOPT_XFERINFOFUNCTION,
+ pakfire_downloader_transfer_update);
+ curl_easy_setopt(transfer->handle, CURLOPT_XFERINFODATA, transfer);
+ curl_easy_setopt(transfer->handle, CURLOPT_NOPROGRESS, 0L);
+
+ // Start progress
+ r = pakfire_progress_start(transfer->progress, transfer->expected_size);
+ if (r)
+ return r;
+
+ return 0;
}
-static int pakfire_downloader_prepare_transfer(struct pakfire_downloader* downloader,
- struct pakfire_transfer* transfer) {
+static int pakfire_downloader_transfer_prepare_tmpfile(struct pakfire_transfer* transfer) {
+ char path[PATH_MAX] = PAKFIRE_TMP_DIR "/pakfire-download.XXXXXX";
+ FILE* f = NULL;
+ int r;
+
+ // Allocate a temporary file
+ f = pakfire_mktemp(path, 0600);
+ if (!f)
+ return -errno;
+
+ // Set this as output
+ r = pakfire_downloader_transfer_set_output(transfer, f);
+ if (r)
+ return r;
+
+ // Store the path to the temporary file
+ return pakfire_string_set(transfer->tempfile, path);
+}
+
+static int pakfire_downloader_transfer_prepare(struct pakfire_downloader* downloader,
+ struct pakfire_transfer* transfer, struct pakfire_progress* progress, int flags) {
+ char url[PATH_MAX];
int r;
// Increment tries
if (r)
return r;
- char* url = pakfire_url_join(transfer->mirror->url, transfer->url);
- if (!url) {
- ERROR(downloader->pakfire, "Error composing download URL: %m\n");
- return 1;
- }
+ r = pakfire_url_join(url, transfer->mirror->url, transfer->url);
+ if (r)
+ return r;
curl_easy_setopt(transfer->handle, CURLOPT_URL, url);
- free(url);
// Use baseurl
} else if (*transfer->baseurl) {
- char* url = pakfire_url_join(transfer->baseurl, transfer->url);
- if (!url) {
- ERROR(downloader->pakfire, "Error composing download URL: %m\n");
- return 1;
- }
+ r = pakfire_url_join(url, transfer->baseurl, transfer->url);
+ if (r)
+ return r;
curl_easy_setopt(transfer->handle, CURLOPT_URL, url);
- free(url);
- // Cannot compose URL
+ // Fail if we could not set the URL
} else {
- ERROR(downloader->pakfire, "Could not set URL\n");
- return 1;
+ ERROR(downloader->pakfire, "Invalid transfer %s\n", transfer->url);
+ return -EINVAL;
}
- // Open a temporary file to write the output to
+ // If we do not have an output file, we will create a temporary file
if (!transfer->f) {
- // Make path for the temporary file (must be on the same file system)
- r = pakfire_string_format(transfer->tempfile, "%s.XXXXXX", transfer->path);
- if (r)
- return 1;
-
- transfer->f = pakfire_mktemp(transfer->tempfile, 0);
- if (!transfer->f) {
- ERROR(downloader->pakfire, "Could not create temporary file for download %s: %m\n",
- transfer->tempfile);
- return 1;
+ r = pakfire_downloader_transfer_prepare_tmpfile(transfer);
+ if (r) {
+ ERROR_ERRNO(transfer->pakfire, r, "Could not open a temporary file: %m\n");
+ return r;
}
}
}
}
- // Write all data to the callback function
- curl_easy_setopt(transfer->handle, CURLOPT_WRITEFUNCTION, pakfire_downloader_write);
- curl_easy_setopt(transfer->handle, CURLOPT_WRITEDATA, transfer);
-
- return 0;
-}
-
-static int pakfire_downloader_update_progress(void* data,
- curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
- struct pakfire_downloader* downloader = (struct pakfire_downloader*)data;
-
- // Set maximum (because this might have changed)
- pakfire_progress_set_max_value(downloader->progress, dltotal);
-
- // Update current value
- return pakfire_progress_update(downloader->progress, dlnow);
-}
-
-static int pakfire_downloader_make_single_progress(
- struct pakfire_downloader* downloader, struct pakfire_transfer* transfer) {
- int r;
-
- // Reset the progress
- pakfire_progress_reset(downloader->progress);
-
- // Set title
- if (*transfer->title) {
- r = pakfire_progress_set_title(downloader->progress, "%s", transfer->title);
- if (r)
- return r;
- }
-
- // Update the progress
- curl_easy_setopt(transfer->handle, CURLOPT_XFERINFOFUNCTION,
- pakfire_downloader_update_progress);
- curl_easy_setopt(transfer->handle, CURLOPT_XFERINFODATA, downloader);
- curl_easy_setopt(transfer->handle, CURLOPT_NOPROGRESS, 0L);
-
- // Start the progress
- r = pakfire_progress_start(downloader->progress, 0);
+ // Setup progress
+ r = pakfire_downloader_transfer_prepare_progress(downloader, transfer, progress, flags);
if (r)
return r;
*/
static int pakfire_downloader_perform(
struct pakfire_downloader* downloader, struct pakfire_transfer* transfer) {
- const int has_progress = !(transfer->flags & PAKFIRE_TRANSFER_NOPROGRESS);
int r;
- // Create progress
- if (has_progress) {
- r = pakfire_downloader_make_single_progress(downloader, transfer);
- if (r)
- return r;
- }
-
AGAIN:
// Prepare the transfer
- r = pakfire_downloader_prepare_transfer(downloader, transfer);
+ r = pakfire_downloader_transfer_prepare(downloader, transfer, NULL, 0);
if (r)
return r;
if (r == EAGAIN)
goto AGAIN;
- // Finish the progress
- if (has_progress) {
- r = pakfire_progress_finish(downloader->progress);
- if (r)
- return r;
+ return r;
+}
+
+int pakfire_downloader_transfer_run(struct pakfire_transfer* transfer, int flags) {
+ struct pakfire_downloader* downloader = transfer->downloader;
+ int r;
+
+AGAIN:
+ // Prepare the transfer
+ r = pakfire_downloader_transfer_prepare(downloader, transfer, NULL, flags);
+ if (r)
+ return r;
+
+ // Perform the action
+ r = curl_easy_perform(transfer->handle);
+
+ // Handle the result
+ r = pakfire_transfer_done(downloader, transfer, r);
+
+ // Repeat the transfer if asked
+ switch (-r) {
+ case EAGAIN:
+ goto AGAIN;
+
+ default:
+ break;
}
return r;
// Perform the download
int r = pakfire_downloader_perform(downloader, transfer);
- pakfire_transfer_free(transfer);
+ pakfire_downloader_transfer_free(transfer);
return r;
}
-int pakfire_downloader_run(struct pakfire_downloader* downloader) {
- struct pakfire_transfer* transfer;
- unsigned int transfers = 0;
+static size_t pakfire_downloader_total_downloadsize(struct pakfire_downloader* downloader) {
+ struct pakfire_transfer* transfer = NULL;
+ size_t size;
+
+ size_t total_size = 0;
+
+ TAILQ_FOREACH(transfer, &downloader->transfers_queued, nodes) {
+ size = pakfire_downloader_transfer_get_size(transfer);
+
+ // Return zero if the size isn't known
+ if (!size)
+ return 0;
+
+ total_size += size;
+ }
+
+ return total_size;
+}
+
+static unsigned int pakfire_downloader_total_queued_transfers(struct pakfire_downloader* downloader) {
+ struct pakfire_transfer* transfer = NULL;
+ unsigned int counter = 0;
+
+ TAILQ_FOREACH(transfer, &downloader->transfers_queued, nodes)
+ counter++;
+
+ return counter;
+}
+
+static int pakfire_downloader_transfer_start(struct pakfire_downloader* downloader,
+ struct pakfire_transfer* transfer, struct pakfire_progress* progress) {
+ int r;
+
+ DEBUG(downloader->pakfire, "Starting transfer %p...\n", transfer);
+ // Prepare the transfer
+ r = pakfire_downloader_transfer_prepare(downloader, transfer, progress, 0);
+ if (r)
+ return r;
+
+ // Add the handle to cURL
+ r = curl_multi_add_handle(downloader->curl, transfer->handle);
+ if (r) {
+ ERROR(downloader->pakfire, "Adding handle failed: %s\n", curl_multi_strerror(r));
+ return r;
+ }
+
+ return r;
+}
+
+static int pakfire_downloader_start_transfers(struct pakfire_downloader* downloader,
+ struct pakfire_progress* progress, unsigned int* running_transfers) {
+ struct pakfire_transfer* transfer = NULL;
int r;
+
+ // Keep running until we have reached our ceiling
+ while (*running_transfers < downloader->parallel) {
+ // We are done if there are no more transfers in the queue
+ if (TAILQ_EMPTY(&downloader->transfers_queued))
+ break;
+
+ // Fetch the next transfer
+ transfer = TAILQ_LAST(&downloader->transfers_queued, transfers_queued);
+ TAILQ_REMOVE(&downloader->transfers_queued, transfer, nodes);
+
+ // Start the transfer
+ r = pakfire_downloader_transfer_start(downloader, transfer, progress);
+ if (r) {
+ TAILQ_INSERT_TAIL(&downloader->transfers_finished, transfer, nodes);
+ return r;
+ }
+
+ TAILQ_INSERT_TAIL(&downloader->transfers_running, transfer, nodes);
+ (*running_transfers)++;
+ }
+
+ return 0;
+}
+
+int pakfire_downloader_run(struct pakfire_downloader* downloader, const char* title) {
+ struct pakfire_progress* progress = NULL;
+ struct pakfire_transfer* transfer = NULL;
+ unsigned int running_transfers = 0;
+ int progress_flags =
+ PAKFIRE_PROGRESS_SHOW_PERCENTAGE |
+ PAKFIRE_PROGRESS_SHOW_ETA;
+ int r = 0;
+
+ CURLMsg* msg = NULL;
int still_running;
int msgs_left = -1;
- int success = 0;
+
+ // Fetch the total downloadsize
+ const size_t downloadsize = pakfire_downloader_total_downloadsize(downloader);
+
+ // If we know the downloadsize, we can show more details
+ if (downloadsize) {
+ progress_flags |=
+ PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED |
+ PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED;
+ } else {
+ progress_flags |=
+ PAKFIRE_PROGRESS_SHOW_COUNTER;
+ }
+
+ // Fetch the total number of queued transfers
+ const unsigned int num_queued_transfers = pakfire_downloader_total_queued_transfers(downloader);
+
+ // Create a new progress indicator
+ r = pakfire_progress_create(&progress, downloader->pakfire, progress_flags, NULL);
+ if (r)
+ goto ERROR;
+
+ // Set the title
+ if (title) {
+ r = pakfire_progress_set_title(progress, title);
+ if (r)
+ goto ERROR;
+ }
+
+ // Start the progress
+ r = pakfire_progress_start(progress, (downloadsize) ? downloadsize : num_queued_transfers);
+ if (r)
+ goto ERROR;
do {
// Make sure that we have up to parallel transfers active
- while (!TAILQ_EMPTY(&downloader->transfers) && transfers < downloader->parallel) {
- // Fetch first item added
- transfer = TAILQ_LAST(&downloader->transfers, transfers);
- TAILQ_REMOVE(&downloader->transfers, transfer, nodes);
-
- // Prepare the transfer
- r = pakfire_downloader_prepare_transfer(downloader, transfer);
- if (r) {
- pakfire_transfer_free(transfer);
- return r;
- }
-
- // Add the handle to cURL
- r = curl_multi_add_handle(downloader->curl, transfer->handle);
- if (r) {
- ERROR(downloader->pakfire, "Adding handle failed\n");
- pakfire_transfer_free(transfer);
- return r;
- }
+ if (!r) {
+ r = pakfire_downloader_start_transfers(downloader, progress, &running_transfers);
+ if (r)
+ goto ERROR;
+ }
- transfers++;
+ // Run cURL
+ r = curl_multi_perform(downloader->curl, &still_running);
+ if (r) {
+ ERROR(downloader->pakfire, "cURL error: %s\n", curl_easy_strerror(r));
+ goto ERROR;
}
- curl_multi_perform(downloader->curl, &still_running);
-
- CURLMsg* msg;
- while ((msg = curl_multi_info_read(downloader->curl, &msgs_left))) {
- if (msg->msg == CURLMSG_DONE) {
- transfers--;
-
- // Update reference to transfer
- curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &transfer);
- curl_multi_remove_handle(downloader->curl, transfer->handle);
-
- // Handle whatever comes after the transfer
- r = pakfire_transfer_done(downloader, transfer, msg->data.result);
- if (r) {
- // We will try another mirror and therefore insert this
- // transfer to be picked up again next
- if (r == EAGAIN) {
- TAILQ_INSERT_TAIL(&downloader->transfers, transfer, nodes);
- continue;
+ for (;;) {
+ // Read the next message
+ msg = curl_multi_info_read(downloader->curl, &msgs_left);
+ if (!msg)
+ break;
+
+ switch (msg->msg) {
+ case CURLMSG_DONE:
+ // Update reference to transfer
+ curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &transfer);
+
+ // Remove the handle
+ curl_multi_remove_handle(downloader->curl, transfer->handle);
+
+ // Remove the transfer from the running list
+ TAILQ_REMOVE(&downloader->transfers_running, transfer, nodes);
+ running_transfers--;
+
+ // Call the done callback
+ r = pakfire_transfer_done(downloader, transfer, msg->data.result);
+ switch (-r) {
+ // If we are asked to try again we will re-queue the transfer
+ case EAGAIN:
+ TAILQ_INSERT_TAIL(&downloader->transfers_queued, transfer, nodes);
+ break;
+
+ // Otherwise this transfer has finished
+ default:
+ TAILQ_INSERT_TAIL(&downloader->transfers_finished, transfer, nodes);
+ if (r)
+ goto ERROR;
+ break;
}
- success = r;
- }
+ // Reset transfer
+ transfer = NULL;
+ break;
- // Destroy transfer
- pakfire_transfer_free(transfer);
- // Log any other messages
- } else {
- ERROR(downloader->pakfire, "cURL reported error %d\n", msg->msg);
+ default:
+ ERROR(downloader->pakfire, "Received unhandled cURL message %d\n", msg->msg);
+ break;
}
}
// Wait a little before going through the loop again
if (still_running)
curl_multi_wait(downloader->curl, NULL, 0, 100, NULL);
- } while (still_running || !TAILQ_EMPTY(&downloader->transfers));
+ } while (still_running || !TAILQ_EMPTY(&downloader->transfers_queued));
- // Success
- return success;
+ // We are finished!
+ r = pakfire_progress_finish(progress);
+ if (r)
+ goto ERROR;
+
+ERROR:
+ // If the downloader was aborted, we need to fail all remaining running transfers
+ while (!TAILQ_EMPTY(&downloader->transfers_running)) {
+ transfer = TAILQ_LAST(&downloader->transfers_running, transfers_running);
+ TAILQ_REMOVE(&downloader->transfers_running, transfer, nodes);
+
+ // Fail the transfer
+ pakfire_transfer_fail(downloader, transfer, 0);
+ }
+
+ if (progress)
+ pakfire_progress_unref(progress);
+
+ return r;
}
struct pakfire_mirrorlist {
// Base URL
char baseurl[PATH_MAX];
+ char expanded_baseurl[PATH_MAX];
// Refresh Interval
time_t refresh;
struct pakfire_key* key;
};
+static const char* pakfire_repo_get_expanded_baseurl(struct pakfire_repo* repo);
+
int pakfire_repo_is_internal(struct pakfire_repo* repo) {
const char* name = pakfire_repo_get_name(repo);
if (!name)
return __pakfire_cache_path(repo->pakfire, path, length, "%s/%s", name, buffer);
}
-static int pakfire_repo_retrieve(
- struct pakfire_repo* repo,
- const char* title,
- const char* url,
- const char* path,
- enum pakfire_digest_types md,
- const unsigned char* expected_digest,
- const size_t expected_digest_length,
- enum pakfire_transfer_flags flags) {
- struct pakfire_downloader* downloader = NULL;
- char* baseurl = NULL;
+/*
+ This is a convenience function to create a transfer with the
+ settings of this repository.
+*/
+static int pakfire_repo_create_transfer(struct pakfire_transfer** transfer,
+ struct pakfire_repo* repo, struct pakfire_downloader* downloader, const char* url) {
+ struct pakfire_mirrorlist* mirrorlist = NULL;
+ struct pakfire_transfer* t = NULL;
+ const char* baseurl = NULL;
int r;
- // Create downloader
- r = pakfire_downloader_create(&downloader, repo->pakfire);
+ // Create a new transfer
+ r = pakfire_downloader_transfer_create(&t, downloader, url);
if (r)
goto ERROR;
- // Prepare the baseurl
- baseurl = pakfire_repo_url_replace(repo, repo->appdata->baseurl);
- if (!baseurl) {
- r = -errno;
- goto ERROR;
+ // Set the baseurl
+ baseurl = pakfire_repo_get_expanded_baseurl(repo);
+ if (baseurl) {
+ r = pakfire_downloader_transfer_set_baseurl(t, baseurl);
+ if (r)
+ goto ERROR;
}
-#if 0
- // Fetch mirrorlist
- struct pakfire_mirrorlist* mirrorlist = pakfire_repo_get_mirrorlist(repo);
-#endif
+ // Set the mirrorlist
+ mirrorlist = pakfire_repo_get_mirrorlist(repo);
+ if (mirrorlist) {
+ r = pakfire_downloader_transfer_set_mirrorlist(t, mirrorlist);
+ if (r)
+ goto ERROR;
+ }
- // Retrieve the database file
- r = pakfire_downloader_retrieve(downloader, baseurl, NULL,
- title, url, path, md, expected_digest, expected_digest_length, flags);
+ // Success
+ *transfer = t;
+ r = 0;
-#if 0
- if (mirrorlist)
- pakfire_mirrorlist_unref(mirrorlist);
-#endif
+ goto CLEANUP;
ERROR:
- if (downloader)
- pakfire_downloader_unref(downloader);
- if (baseurl)
- free(baseurl);
+ if (t)
+ pakfire_downloader_transfer_unref(t);
+
+CLEANUP:
+ if (mirrorlist)
+ pakfire_mirrorlist_unref(mirrorlist);
return r;
}
static int pakfire_repo_download_database(struct pakfire_repo* repo,
const char* filename, const char* path) {
+ struct pakfire_downloader* downloader = NULL;
+ struct pakfire_transfer* transfer = NULL;
char title[NAME_MAX];
char url[PATH_MAX];
int r;
if (r)
return r;
- return pakfire_repo_retrieve(repo, title, url, path, 0, NULL, 0, 0);
+ // Create a downloader
+ r = pakfire_downloader_create(&downloader, repo->pakfire);
+ if (r)
+ goto ERROR;
+
+ // Create a new transfer
+ r = pakfire_repo_create_transfer(&transfer, repo, downloader, url);
+ if (r)
+ goto ERROR;
+
+ // Set title
+ r = pakfire_downloader_transfer_set_title(transfer, title);
+ if (r)
+ goto ERROR;
+
+ // Set path
+ r = pakfire_downloader_transfer_set_target(transfer, path);
+ if (r)
+ goto ERROR;
+
+ // Run the transfer
+ r = pakfire_downloader_transfer_run(transfer, 0);
+ if (r)
+ goto ERROR;
+
+ERROR:
+ if (transfer)
+ pakfire_downloader_transfer_unref(transfer);
+ if (downloader)
+ pakfire_downloader_unref(downloader);
+
+ return r;
}
static int pakfire_repo_read_database(struct pakfire_repo* repo, const char* path) {
}
static int pakfire_repo_refresh_mirrorlist(struct pakfire_repo* repo, const int force) {
+ struct pakfire_downloader* downloader = NULL;
+ struct pakfire_transfer* transfer = NULL;
char* url = NULL;
int r;
goto ERROR;
}
- r = pakfire_repo_retrieve(repo, NULL, url, repo->appdata->mirrorlist,
- 0, NULL, 0, PAKFIRE_TRANSFER_NOPROGRESS);
+ // Create a new downloader
+ r = pakfire_downloader_create(&downloader, repo->pakfire);
+ if (r)
+ goto ERROR;
+
+ // Create a new transfer
+ r = pakfire_repo_create_transfer(&transfer, repo, downloader, url);
+ if (r)
+ goto ERROR;
+
+ // Run the transfer
+ r = pakfire_downloader_transfer_run(transfer, PAKFIRE_TRANSFER_NO_PROGRESS);
+ if (r)
+ goto ERROR;
ERROR:
+ if (transfer)
+ pakfire_downloader_transfer_unref(transfer);
+ if (downloader)
+ pakfire_downloader_unref(downloader);
if (url)
free(url);
}
static int pakfire_repo_download_metadata(struct pakfire_repo* repo, const char* path, int force) {
+ struct pakfire_downloader* downloader = NULL;
+ struct pakfire_transfer* transfer = NULL;
+ int r;
+
// Local repositories don't need to download metadata
if (pakfire_repo_is_local(repo))
return 0;
}
}
- return pakfire_repo_retrieve(repo, NULL, "repodata/repomd.json", path,
- 0, NULL, 0, PAKFIRE_TRANSFER_NOPROGRESS);
+ // Create a downloader
+ r = pakfire_downloader_create(&downloader, repo->pakfire);
+ if (r)
+ goto ERROR;
+
+ // Create a new transfer
+ r = pakfire_repo_create_transfer(&transfer, repo, downloader, "repodata/repomd.json");
+ if (r)
+ goto ERROR;
+
+ // Set the target path
+ r = pakfire_downloader_transfer_set_target(transfer, path);
+ if (r)
+ goto ERROR;
+
+ // Perform the transfer
+ r = pakfire_downloader_transfer_run(transfer, PAKFIRE_TRANSFER_NO_PROGRESS);
+ if (r)
+ goto ERROR;
+
+ERROR:
+ if (transfer)
+ pakfire_downloader_transfer_unref(transfer);
+ if (downloader)
+ pakfire_downloader_unref(downloader);
+
+ return r;
}
static int pakfire_repo_refresh_metadata(struct pakfire_repo* repo, const int force) {
return repo->appdata->baseurl;
}
+static const char* pakfire_repo_get_expanded_baseurl(struct pakfire_repo* repo) {
+ char* url = NULL;
+ int r;
+
+ // Return NULL if baseurl isn't set
+ if (!*repo->appdata->baseurl)
+ return NULL;
+
+ if (!*repo->appdata->expanded_baseurl) {
+ url = pakfire_repo_url_replace(repo, repo->appdata->baseurl);
+ if (!url)
+ goto ERROR;
+
+ // Store the result
+ r = pakfire_string_set(repo->appdata->expanded_baseurl, url);
+ if (r)
+ goto ERROR;
+ }
+
+ERROR:
+ if (url)
+ free(url);
+
+ return (*repo->appdata->expanded_baseurl) ? repo->appdata->expanded_baseurl : NULL;
+}
+
PAKFIRE_EXPORT int pakfire_repo_set_baseurl(struct pakfire_repo* repo, const char* baseurl) {
int r;
if (r)
return r;
+ // Invalidate the expanded URL
+ *repo->appdata->expanded_baseurl = '\0';
+
// Update sub-priority
pakfire_repo_update_subpriority(repo);
return (r == 0);
}
+int pakfire_repo_download_package(struct pakfire_repo* repo, struct pakfire_downloader* downloader,
+ struct pakfire_package* pkg, struct pakfire_transfer** transfer) {
+ struct pakfire_transfer* t = NULL;
+ const unsigned char* digest = NULL;
+ const char* cache_path = NULL;
+ const char* nevra = NULL;
+ const char* url = NULL;
+ int r = 1;
+
+ // Fetch nevra
+ nevra = pakfire_package_get_string(pkg, PAKFIRE_PKG_NEVRA);
+ if (!nevra)
+ goto ERROR;
+
+ // Where do we store the package?
+ cache_path = pakfire_package_get_string(pkg, PAKFIRE_PKG_CACHE_PATH);
+ if (!cache_path) {
+ ERROR(repo->pakfire, "Could not retrieve package cache path for %s: %m\n", nevra);
+ goto ERROR;
+ }
+
+ // Where do we find the package on the server?
+ url = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH);
+ if (!url) {
+ ERROR(repo->pakfire, "Could not retrieve package path for %s: %m\n", nevra);
+ goto ERROR;
+ }
+
+ enum pakfire_digest_types digest_type = 0;
+ size_t digest_length = 0;
+
+ // Retrieve package digest
+ digest = pakfire_package_get_digest(pkg, &digest_type, &digest_length);
+ if (!digest) {
+ ERROR(repo->pakfire, "Package %s has no digest set: %m\n", nevra);
+ goto ERROR;
+ }
+
+ // Create a new transfer
+ r = pakfire_repo_create_transfer(&t, repo, downloader, url);
+ if (r)
+ goto ERROR;
+
+ // Set title
+ r = pakfire_downloader_transfer_set_title(t, nevra);
+ if (r)
+ goto ERROR;
+
+ // Fetch downloadsize
+ size_t downloadsize = pakfire_package_get_num(pkg, PAKFIRE_PKG_DOWNLOADSIZE, 0);
+
+ // Set size
+ if (downloadsize > 0) {
+ r = pakfire_downloader_transfer_set_size(t, downloadsize);
+ if (r)
+ goto ERROR;
+ }
+
+ // Set target
+ r = pakfire_downloader_transfer_set_target(t, cache_path);
+ if (r)
+ goto ERROR;
+
+ // Set digest
+ r = pakfire_downloader_transfer_verify_digest(t, digest_type, digest, digest_length);
+ if (r)
+ goto ERROR;
+
+ // Success
+ *transfer = t;
+
+ return 0;
+
+ERROR:
+ if (t)
+ pakfire_downloader_transfer_unref(t);
+
+ return r;
+}
+
static int pakfire_repo_download(struct pakfire_repo* repo, const char* url,
struct pakfire_package** package) {
struct pakfire_downloader* downloader = NULL;
+ struct pakfire_transfer* transfer = NULL;
FILE* f = NULL;
int r;
if (r)
goto ERROR;
- // Download the file
- r = pakfire_downloader_retrieve(downloader, NULL, NULL, NULL,
- url, path, 0, NULL, 0, PAKFIRE_TRANSFER_NOTEMP);
+ // Create a new transfer
+ r = pakfire_downloader_transfer_create(&transfer, downloader, url);
+ if (r)
+ goto ERROR;
+
+ // Set destination
+ r = pakfire_downloader_transfer_set_output(transfer, f);
+ if (r)
+ goto ERROR;
+
+ // Run the download
+ r = pakfire_downloader_transfer_run(transfer, 0);
if (r)
goto ERROR;
goto ERROR;
ERROR:
+ if (transfer)
+ pakfire_downloader_transfer_unref(transfer);
if (downloader)
pakfire_downloader_unref(downloader);
if (f)
}
// Download the package if we got given a URL
- if (pakfire_string_is_url(path))
- return pakfire_repo_download(repo, path, package);
+ if (pakfire_string_is_url(path)) {
+ r = pakfire_repo_download(repo, path, package);
+ if (r)
+ ERROR(repo->pakfire, "Could not download %s\n", path);
+
+ return r;
+ }
// Try to open the archive
r = pakfire_archive_open(&archive, repo->pakfire, path);
projects / pakfire.git / commitdiff
