From: Michael Tremer Date: Wed, 10 Mar 2021 19:20:16 +0000 (+0000) Subject: dist: Implement downloader to download source X-Git-Tag: 0.9.28~1285^2~575 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=62b60e379b06da30c5bb2a78642d1c340e18fb43;p=pakfire.git dist: Implement downloader to download source Signed-off-by: Michael Tremer --- diff --git a/src/libpakfire/dist.c b/src/libpakfire/dist.c index 38b97a674..92979b0d3 100644 --- a/src/libpakfire/dist.c +++ b/src/libpakfire/dist.c @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -38,46 +39,56 @@ static int pakfire_dist_download_source(Pakfire pakfire, const char* filename, const char* cache_path) { + struct pakfire_downloader* downloader; + DEBUG(pakfire, "Downloading %s...\n", filename); - return 1; + int r = pakfire_downloader_create(&downloader, pakfire); + if (r) { + ERROR(pakfire, "Could not initialize downloader\n"); + return r; + } + + // Add download + r = pakfire_downloader_add(downloader, filename, cache_path); + if (r) + goto ERROR; + + // Perform download + r = pakfire_downloader_run(downloader); + if (r) + goto ERROR; + + // Success + r = 0; + +ERROR: + pakfire_downloader_unref(downloader); + + return r; } static int pakfire_dist_add_source(Pakfire pakfire, struct pakfire_packager* packager, PakfirePackage pkg, const char* filename) { int r; char archive_path[PATH_MAX]; - char path[PATH_MAX]; + char cache_path[PATH_MAX]; const char* name = pakfire_package_get_name(pkg); - snprintf(path, sizeof(path) - 1, "sources/%s/%s", name, filename); snprintf(archive_path, sizeof(archive_path) - 1, "files/%s", filename); + snprintf(cache_path, sizeof(cache_path) - 1, "%s/sources/%s/%s", + pakfire_get_cache_path(pakfire), name, filename); // Download the file if it does not exist in the cache - if (pakfire_cache_access(pakfire, path, R_OK)) { - r = pakfire_dist_download_source(pakfire, filename, path); + if (access(cache_path, R_OK) != 0) { + r = pakfire_dist_download_source(pakfire, filename, cache_path); if (r) return r; } - // Make path in cache - char* cache_abspath = pakfire_get_cache_path(pakfire, path); - if (!cache_abspath) - return 1; - // Add file to package - r = pakfire_packager_add(packager, cache_abspath, archive_path); - if (r) - goto ERROR; - - // Success - r = 0; - -ERROR: - free(cache_abspath); - - return r; + return pakfire_packager_add(packager, cache_path, archive_path); } static int pakfire_dist_add_sources(Pakfire pakfire, struct pakfire_packager* packager, diff --git a/src/libpakfire/downloader.c b/src/libpakfire/downloader.c index a158fac5d..61f7a550b 100644 --- a/src/libpakfire/downloader.c +++ b/src/libpakfire/downloader.c @@ -20,17 +20,75 @@ #include #include +#include + +#include #include +#include #include #include +#include + +// The number of concurrent downloads +#define MAX_PARALLEL 4 + +/* + Count how many downloaders are using cURL and free global resources when no + downloaders are being left. +*/ +static int curl_initialized = 0; struct pakfire_downloader { Pakfire pakfire; int nrefs; + + // cURL multi handle + CURLM* curl; +}; + +struct pakfire_downloader_transfer { + CURL* handle; + + // Where do we write the result to? + char path[PATH_MAX]; + + // Temporary file + char tempfile[PATH_MAX]; + FILE* f; }; +static int pakfire_downloader_setup_curl(struct pakfire_downloader* downloader) { + // Globally initialise cURL + if (!curl_initialized++) { + int r = curl_global_init(CURL_GLOBAL_ALL); + if (r) { + ERROR(downloader->pakfire, "Could not setup cURL: %d\n", r); + return r; + } + } + + // Create a new multi handle + downloader->curl = curl_multi_init(); + if (!downloader->curl) { + ERROR(downloader->pakfire, "Could not create cURL multi handle\n"); + return 1; + } + + // Limit the amount of parallel downloads + curl_multi_setopt(downloader->curl, CURLMOPT_MAXCONNECTS, (long)MAX_PARALLEL); + + return 0; +} + static void pakfire_downloader_free(struct pakfire_downloader* downloader) { + if (downloader->curl) + curl_multi_cleanup(downloader->curl); + + // Cleanup global stuff after all downloader instances have been freed + if (!--curl_initialized) + curl_global_cleanup(); + pakfire_unref(downloader->pakfire); free(downloader); } @@ -46,9 +104,19 @@ int pakfire_downloader_create(struct pakfire_downloader** downloader, Pakfire pa // Initialize reference counting d->nrefs = 1; + // Setup cURL + int r = pakfire_downloader_setup_curl(d); + if (r) + goto ERROR; + *downloader = d; return 0; + +ERROR: + pakfire_downloader_free(d); + + return r; } struct pakfire_downloader* pakfire_downloader_ref(struct pakfire_downloader* downloader) { @@ -65,10 +133,169 @@ struct pakfire_downloader* pakfire_downloader_unref(struct pakfire_downloader* d return NULL; } -int pakfire_downloader_add(struct pakfire_downloader* downloader, const char* url) { +#ifdef ENABLE_DEBUG +static int debug_callback(CURL *handle, curl_infotype type, + char* data, size_t size, void* private) { + Pakfire pakfire = (Pakfire)private; + + switch (type) { + case CURLINFO_TEXT: + DEBUG(pakfire, "cURL: %s", data); + break; + + // Log headers + case CURLINFO_HEADER_IN: + DEBUG(pakfire, "cURL: < %s", data); + break; + + case CURLINFO_HEADER_OUT: + DEBUG(pakfire, "cURL: > %s", data); + break; + + // Ignore everything else + default: + break; + } + return 0; } +#endif + +static void pakfire_downloader_transfer_free(struct pakfire_downloader_transfer* transfer) { + if (transfer->handle) + curl_easy_cleanup(transfer->handle); + + // Close temporary file + if (transfer->f) { + unlink(transfer->tempfile); + fclose(transfer->f); + } + + free(transfer); +} + +static struct pakfire_downloader_transfer* pakfire_downloader_create_transfer( + struct pakfire_downloader* downloader, const char* path) { + struct pakfire_downloader_transfer* transfer = calloc(1, sizeof(*transfer)); + if (!transfer) + return NULL; + + // Copy path + snprintf(transfer->path, sizeof(transfer->path) - 1, "%s", path); + + // Make path for the temporary file (must be on the same file system) + snprintf(transfer->tempfile, sizeof(transfer->tempfile) - 1, "%s.XXXXXX", path); + + // Allocate handle + transfer->handle = curl_easy_init(); + if (!transfer->handle) + goto ERROR; + + // Enable logging/debugging + curl_easy_setopt(transfer->handle, CURLOPT_VERBOSE, 1); + +#ifdef ENABLE_DEBUG + curl_easy_setopt(transfer->handle, CURLOPT_DEBUGFUNCTION, debug_callback); + curl_easy_setopt(transfer->handle, CURLOPT_DEBUGDATA, downloader->pakfire); +#endif + + // Reference back to this transfer + curl_easy_setopt(transfer->handle, CURLOPT_PRIVATE, transfer); + + // Follow any redirects + curl_easy_setopt(transfer->handle, CURLOPT_FOLLOWLOCATION, 1); + + // Open a temporary file to write the output to + transfer->f = pakfire_mktemp(transfer->tempfile); + if (!transfer->f) { + ERROR(downloader->pakfire, "Could not create temporary file for download: %s\n", + strerror(errno)); + goto ERROR; + } + + curl_easy_setopt(transfer->handle, CURLOPT_WRITEDATA, transfer->f); + + return transfer; + +ERROR: + pakfire_downloader_transfer_free(transfer); + + return NULL; +} + +static int pakfire_downloader_transfer_done(struct pakfire_downloader* downloader, + struct pakfire_downloader_transfer* transfer, CURLMsg* msg) { + int r; + + DEBUG(downloader->pakfire, "cURL transfer done: %d - %s\n", + msg->data.result, curl_easy_strerror(msg->data.result)); + + // Move the temporary file to its destination + r = link(transfer->tempfile, transfer->path); + if (r) { + ERROR(downloader->pakfire, "Could not link destination file %s: %s\n", + transfer->path, strerror(errno)); + return r; + } + + return 0; +} + +int pakfire_downloader_add(struct pakfire_downloader* downloader, + const char* url, const char* path) { + DEBUG(downloader->pakfire, "Adding download of %s\n", url); + + struct pakfire_downloader_transfer* transfer = + pakfire_downloader_create_transfer(downloader, path); + if (!transfer) + return 1; + + // Set URL + curl_easy_setopt(transfer->handle, CURLOPT_URL, url); + + // Finally, add the handle to the queue + int r = curl_multi_add_handle(downloader->curl, transfer->handle); + if (r) { + ERROR(downloader->pakfire, "Adding handler failed\n"); + pakfire_downloader_transfer_free(transfer); + } + + return r; +} int pakfire_downloader_run(struct pakfire_downloader* downloader) { + int still_running; + int msgs_left = -1; + + do { + curl_multi_perform(downloader->curl, &still_running); + + CURLMsg* msg; + while ((msg = curl_multi_info_read(downloader->curl, &msgs_left))) { + struct pakfire_downloader_transfer* transfer; + + if (msg->msg == CURLMSG_DONE) { + // Update reference to transfer + curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &transfer); + + // Handle whatever comes after the transfer + pakfire_downloader_transfer_done(downloader, transfer, msg); + + // Remove and destroy the handle + curl_multi_remove_handle(downloader->curl, transfer->handle); + pakfire_downloader_transfer_free(transfer); + + // Log any other messages + } else { + ERROR(downloader->pakfire, "cURL reported error %d\n", msg->msg); + } + } + + // Wait a little before going through the loop again + if (still_running) + curl_multi_wait(downloader->curl, NULL, 0, 100, NULL); + } while (still_running); + + // Success return 0; } diff --git a/src/libpakfire/include/pakfire/downloader.h b/src/libpakfire/include/pakfire/downloader.h index 88d02f9c7..78ec59ac8 100644 --- a/src/libpakfire/include/pakfire/downloader.h +++ b/src/libpakfire/include/pakfire/downloader.h @@ -32,7 +32,8 @@ int pakfire_downloader_create(struct pakfire_downloader** downloader, Pakfire pa struct pakfire_downloader* pakfire_downloader_ref(struct pakfire_downloader* downloader); struct pakfire_downloader* pakfire_downloader_unref(struct pakfire_downloader* downloader); -int pakfire_downloader_add(struct pakfire_downloader* downloader, const char* url); +int pakfire_downloader_add(struct pakfire_downloader* downloader, + const char* url, const char* path); int pakfire_downloader_run(struct pakfire_downloader* downloader); diff --git a/src/libpakfire/include/pakfire/util.h b/src/libpakfire/include/pakfire/util.h index 2deb4faad..f91fb5743 100644 --- a/src/libpakfire/include/pakfire/util.h +++ b/src/libpakfire/include/pakfire/util.h @@ -63,6 +63,8 @@ char* pakfire_lstrip(const char* s); char* pakfire_hexlify(const char* digest, const size_t length); +FILE* pakfire_mktemp(char* path); + #endif #endif /* PAKFIRE_UTIL_H */ diff --git a/src/libpakfire/pakfire.c b/src/libpakfire/pakfire.c index da91f0385..6d5c5d4d2 100644 --- a/src/libpakfire/pakfire.c +++ b/src/libpakfire/pakfire.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include diff --git a/src/libpakfire/util.c b/src/libpakfire/util.c index e4daba7f3..b599e3a2f 100644 --- a/src/libpakfire/util.c +++ b/src/libpakfire/util.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -496,3 +497,44 @@ char* pakfire_hexlify(const char* digest, const size_t length) { return s; } + +static int pakfire_mkparentdir(const char* path) { + int r; + + char* dirname = pakfire_dirname(path); + if (!dirname) + return 1; + + // We have arrived at the top of the tree + if (*dirname == '.' || *dirname == '/') + return 0; + + // Ensure the parent directory exists + r = pakfire_mkparentdir(dirname); + if (r) { + if (errno == EEXIST) + r = 0; + + goto END; + } + + // Create this directory + r = mkdir(dirname, 0); + +END: + free(dirname); + + return r; +} + +FILE* pakfire_mktemp(char* path) { + pakfire_mkparentdir(path); + + // Create a temporary result file + int fd = mkostemp(path, O_CLOEXEC); + if (fd < 0) + return NULL; + + // Re-open as file handle + return fdopen(fd, "w+"); +}