dist: Implement downloader to download source

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/libpakfire/dist.c b/src/libpakfire/dist.c
index 38b97a6748410f7e5e62ecb34f4937afd6a48a5e..92979b0d352c66a2345657123f66c7f5ee331978 100644 (file)
--- a/src/libpakfire/dist.c
+++ b/src/libpakfire/dist.c
@@ -26,6 +26,7 @@
 #include <stdlib.h>
 
 #include <pakfire/dist.h>
+#include <pakfire/downloader.h>
 #include <pakfire/logging.h>
 #include <pakfire/package.h>
 #include <pakfire/packager.h>
@@ -38,46 +39,56 @@
 
 static int pakfire_dist_download_source(Pakfire pakfire, const char* filename,
                const char* cache_path) {
+       struct pakfire_downloader* downloader;
+
        DEBUG(pakfire, "Downloading %s...\n", filename);
 
-       return 1;
+       int r = pakfire_downloader_create(&downloader, pakfire);
+       if (r) {
+               ERROR(pakfire, "Could not initialize downloader\n");
+               return r;
+       }
+
+       // Add download
+       r = pakfire_downloader_add(downloader, filename, cache_path);
+       if (r)
+               goto ERROR;
+
+       // Perform download
+       r = pakfire_downloader_run(downloader);
+       if (r)
+               goto ERROR;
+
+       // Success
+       r = 0;
+
+ERROR:
+       pakfire_downloader_unref(downloader);
+
+       return r;
 }
 
 static int pakfire_dist_add_source(Pakfire pakfire, struct pakfire_packager* packager,
                PakfirePackage pkg, const char* filename) {
        int r;
        char archive_path[PATH_MAX];
-       char path[PATH_MAX];
+       char cache_path[PATH_MAX];
 
        const char* name = pakfire_package_get_name(pkg);
 
-       snprintf(path, sizeof(path) - 1, "sources/%s/%s", name, filename);
        snprintf(archive_path, sizeof(archive_path) - 1, "files/%s", filename);
+       snprintf(cache_path, sizeof(cache_path) - 1, "%s/sources/%s/%s",
+               pakfire_get_cache_path(pakfire), name, filename);
 
        // Download the file if it does not exist in the cache
-       if (pakfire_cache_access(pakfire, path, R_OK)) {
-               r = pakfire_dist_download_source(pakfire, filename, path);
+       if (access(cache_path, R_OK) != 0) {
+               r = pakfire_dist_download_source(pakfire, filename, cache_path);
                if (r)
                        return r;
        }
 
-       // Make path in cache
-       char* cache_abspath = pakfire_get_cache_path(pakfire, path);
-       if (!cache_abspath)
-               return 1;
-
        // Add file to package
-       r = pakfire_packager_add(packager, cache_abspath, archive_path);
-       if (r)
-               goto ERROR;
-
-       // Success
-       r = 0;
-
-ERROR:
-       free(cache_abspath);
-
-       return r;
+       return pakfire_packager_add(packager, cache_path, archive_path);
 }
 
 static int pakfire_dist_add_sources(Pakfire pakfire, struct pakfire_packager* packager,

diff --git a/src/libpakfire/downloader.c b/src/libpakfire/downloader.c
index a158fac5d14526b48e61ceb435998b244d900340..61f7a550b7f142bd291b1551f0bef971a313e916 100644 (file)
--- a/src/libpakfire/downloader.c
+++ b/src/libpakfire/downloader.c
@@ -20,17 +20,75 @@
 
 #include <errno.h>
 #include <stdlib.h>
+#include <unistd.h>
+
+#include <curl/curl.h>
 
 #include <pakfire/downloader.h>
+#include <pakfire/logging.h>
 #include <pakfire/pakfire.h>
 #include <pakfire/types.h>
+#include <pakfire/util.h>
+
+// The number of concurrent downloads
+#define MAX_PARALLEL                   4
+
+/*
+       Count how many downloaders are using cURL and free global resources when no
+       downloaders are being left.
+*/
+static int curl_initialized = 0;
 
 struct pakfire_downloader {
        Pakfire pakfire;
        int nrefs;
+
+       // cURL multi handle
+       CURLM* curl;
+};
+
+struct pakfire_downloader_transfer {
+       CURL* handle;
+
+       // Where do we write the result to?
+       char path[PATH_MAX];
+
+       // Temporary file
+       char tempfile[PATH_MAX];
+       FILE* f;
 };
 
+static int pakfire_downloader_setup_curl(struct pakfire_downloader* downloader) {
+       // Globally initialise cURL
+       if (!curl_initialized++) {
+               int r = curl_global_init(CURL_GLOBAL_ALL);
+               if (r) {
+                       ERROR(downloader->pakfire, "Could not setup cURL: %d\n", r);
+                       return r;
+               }
+       }
+
+       // Create a new multi handle
+       downloader->curl = curl_multi_init();
+       if (!downloader->curl) {
+               ERROR(downloader->pakfire, "Could not create cURL multi handle\n");
+               return 1;
+       }
+
+       // Limit the amount of parallel downloads
+       curl_multi_setopt(downloader->curl, CURLMOPT_MAXCONNECTS, (long)MAX_PARALLEL);
+
+       return 0;
+}
+
 static void pakfire_downloader_free(struct pakfire_downloader* downloader) {
+       if (downloader->curl)
+               curl_multi_cleanup(downloader->curl);
+
+       // Cleanup global stuff after all downloader instances have been freed
+       if (!--curl_initialized)
+               curl_global_cleanup();
+
        pakfire_unref(downloader->pakfire);
        free(downloader);
 }
@@ -46,9 +104,19 @@ int pakfire_downloader_create(struct pakfire_downloader** downloader, Pakfire pa
        // Initialize reference counting
        d->nrefs = 1;
 
+       // Setup cURL
+       int r = pakfire_downloader_setup_curl(d);
+       if (r)
+               goto ERROR;
+
        *downloader = d;
 
        return 0;
+
+ERROR:
+       pakfire_downloader_free(d);
+
+       return r;
 }
 
 struct pakfire_downloader* pakfire_downloader_ref(struct pakfire_downloader* downloader) {
@@ -65,10 +133,169 @@ struct pakfire_downloader* pakfire_downloader_unref(struct pakfire_downloader* d
        return NULL;
 }
 
-int pakfire_downloader_add(struct pakfire_downloader* downloader, const char* url) {
+#ifdef ENABLE_DEBUG
+static int debug_callback(CURL *handle, curl_infotype type,
+               char* data, size_t size, void* private) {
+       Pakfire pakfire = (Pakfire)private;
+
+       switch (type) {
+               case CURLINFO_TEXT:
+                       DEBUG(pakfire, "cURL: %s", data);
+                       break;
+
+               // Log headers
+               case CURLINFO_HEADER_IN:
+                       DEBUG(pakfire, "cURL: < %s", data);
+                       break;
+
+               case CURLINFO_HEADER_OUT:
+                       DEBUG(pakfire, "cURL: > %s", data);
+                       break;
+
+               // Ignore everything else
+               default:
+                       break;
+       }
+
        return 0;
 }
+#endif
+
+static void pakfire_downloader_transfer_free(struct pakfire_downloader_transfer* transfer) {
+       if (transfer->handle)
+               curl_easy_cleanup(transfer->handle);
+
+       // Close temporary file
+       if (transfer->f) {
+               unlink(transfer->tempfile);
+               fclose(transfer->f);
+       }
+
+       free(transfer);
+}
+
+static struct pakfire_downloader_transfer* pakfire_downloader_create_transfer(
+               struct pakfire_downloader* downloader, const char* path) {
+       struct pakfire_downloader_transfer* transfer = calloc(1, sizeof(*transfer));
+       if (!transfer)
+               return NULL;
+
+       // Copy path
+       snprintf(transfer->path, sizeof(transfer->path) - 1, "%s", path);
+
+       // Make path for the temporary file (must be on the same file system)
+       snprintf(transfer->tempfile, sizeof(transfer->tempfile) - 1, "%s.XXXXXX", path);
+
+       // Allocate handle
+       transfer->handle = curl_easy_init();
+       if (!transfer->handle)
+               goto ERROR;
+
+       // Enable logging/debugging
+       curl_easy_setopt(transfer->handle, CURLOPT_VERBOSE, 1);
+
+#ifdef ENABLE_DEBUG
+       curl_easy_setopt(transfer->handle, CURLOPT_DEBUGFUNCTION, debug_callback);
+       curl_easy_setopt(transfer->handle, CURLOPT_DEBUGDATA, downloader->pakfire);
+#endif
+
+       // Reference back to this transfer
+       curl_easy_setopt(transfer->handle, CURLOPT_PRIVATE, transfer);
+
+       // Follow any redirects
+       curl_easy_setopt(transfer->handle, CURLOPT_FOLLOWLOCATION, 1);
+
+       // Open a temporary file to write the output to
+       transfer->f = pakfire_mktemp(transfer->tempfile);
+       if (!transfer->f) {
+               ERROR(downloader->pakfire, "Could not create temporary file for download: %s\n",
+                       strerror(errno));
+               goto ERROR;
+       }
+
+       curl_easy_setopt(transfer->handle, CURLOPT_WRITEDATA, transfer->f);
+
+       return transfer;
+
+ERROR:
+       pakfire_downloader_transfer_free(transfer);
+
+       return NULL;
+}
+
+static int pakfire_downloader_transfer_done(struct pakfire_downloader* downloader,
+               struct pakfire_downloader_transfer* transfer, CURLMsg* msg) {
+       int r;
+
+       DEBUG(downloader->pakfire, "cURL transfer done: %d - %s\n",
+               msg->data.result, curl_easy_strerror(msg->data.result));
+
+       // Move the temporary file to its destination
+       r = link(transfer->tempfile, transfer->path);
+       if (r) {
+               ERROR(downloader->pakfire, "Could not link destination file %s: %s\n",
+                       transfer->path, strerror(errno));
+               return r;
+       }
+
+       return 0;
+}
+
+int pakfire_downloader_add(struct pakfire_downloader* downloader,
+               const char* url, const char* path) {
+       DEBUG(downloader->pakfire, "Adding download of %s\n", url);
+
+       struct pakfire_downloader_transfer* transfer =
+               pakfire_downloader_create_transfer(downloader, path);
+       if (!transfer)
+               return 1;
+
+       // Set URL
+       curl_easy_setopt(transfer->handle, CURLOPT_URL, url);
+
+       // Finally, add the handle to the queue
+       int r = curl_multi_add_handle(downloader->curl, transfer->handle);
+       if (r) {
+               ERROR(downloader->pakfire, "Adding handler failed\n");
+               pakfire_downloader_transfer_free(transfer);
+       }
+
+       return r;
+}
 
 int pakfire_downloader_run(struct pakfire_downloader* downloader) {
+       int still_running;
+       int msgs_left = -1;
+
+       do {
+               curl_multi_perform(downloader->curl, &still_running);
+
+               CURLMsg* msg;
+               while ((msg = curl_multi_info_read(downloader->curl, &msgs_left))) {
+                       struct pakfire_downloader_transfer* transfer;
+
+                       if (msg->msg == CURLMSG_DONE) {
+                               // Update reference to transfer
+                               curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &transfer);
+
+                               // Handle whatever comes after the transfer
+                               pakfire_downloader_transfer_done(downloader, transfer, msg);
+
+                               // Remove and destroy the handle
+                               curl_multi_remove_handle(downloader->curl, transfer->handle);
+                               pakfire_downloader_transfer_free(transfer);
+
+                       // Log any other messages
+                       } else {
+                               ERROR(downloader->pakfire, "cURL reported error %d\n", msg->msg);
+                       }
+               }
+
+               // Wait a little before going through the loop again
+               if (still_running)
+                       curl_multi_wait(downloader->curl, NULL, 0, 100, NULL);
+       } while (still_running);
+
+       // Success
        return 0;
 }

diff --git a/src/libpakfire/include/pakfire/downloader.h b/src/libpakfire/include/pakfire/downloader.h
index 88d02f9c7d91db298c718847b81defbcdfa3a9eb..78ec59ac8bc236a994c3f310392e39e0f540416e 100644 (file)
--- a/src/libpakfire/include/pakfire/downloader.h
+++ b/src/libpakfire/include/pakfire/downloader.h
@@ -32,7 +32,8 @@ int pakfire_downloader_create(struct pakfire_downloader** downloader, Pakfire pa
 struct pakfire_downloader* pakfire_downloader_ref(struct pakfire_downloader* downloader);
 struct pakfire_downloader* pakfire_downloader_unref(struct pakfire_downloader* downloader);
 
-int pakfire_downloader_add(struct pakfire_downloader* downloader, const char* url);
+int pakfire_downloader_add(struct pakfire_downloader* downloader,
+       const char* url, const char* path);
 
 int pakfire_downloader_run(struct pakfire_downloader* downloader);
 

diff --git a/src/libpakfire/include/pakfire/util.h b/src/libpakfire/include/pakfire/util.h
index 2deb4faad3a052c9f459c72374db1282428c4266..f91fb5743d75d8e3e3007de9dffabec3fd5d2964 100644 (file)
--- a/src/libpakfire/include/pakfire/util.h
+++ b/src/libpakfire/include/pakfire/util.h
@@ -63,6 +63,8 @@ char* pakfire_lstrip(const char* s);
 
 char* pakfire_hexlify(const char* digest, const size_t length);
 
+FILE* pakfire_mktemp(char* path);
+
 #endif
 
 #endif /* PAKFIRE_UTIL_H */

diff --git a/src/libpakfire/pakfire.c b/src/libpakfire/pakfire.c
index da91f0385671d57c486793439a5f56ebe5327745..6d5c5d4d282b67708463a98e5e9e8181b732f623 100644 (file)
--- a/src/libpakfire/pakfire.c
+++ b/src/libpakfire/pakfire.c
@@ -22,6 +22,7 @@
 #include <errno.h>
 #include <ftw.h>
 #include <glob.h>
+#include <linux/limits.h>
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>

diff --git a/src/libpakfire/util.c b/src/libpakfire/util.c
index e4daba7f397c9b67194ee44bc46629500f93cc1e..b599e3a2fea7665d46d8cc8a811f546e39afbffb 100644 (file)
--- a/src/libpakfire/util.c
+++ b/src/libpakfire/util.c
@@ -20,6 +20,7 @@
 
 #include <ctype.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <libgen.h>
 #include <math.h>
 #include <stddef.h>
@@ -496,3 +497,44 @@ char* pakfire_hexlify(const char* digest, const size_t length) {
 
        return s;
 }
+
+static int pakfire_mkparentdir(const char* path) {
+       int r;
+
+       char* dirname = pakfire_dirname(path);
+       if (!dirname)
+               return 1;
+
+       // We have arrived at the top of the tree
+       if (*dirname == '.' || *dirname == '/')
+               return 0;
+
+       // Ensure the parent directory exists
+       r = pakfire_mkparentdir(dirname);
+       if (r) {
+               if (errno == EEXIST)
+                       r = 0;
+
+               goto END;
+       }
+
+       // Create this directory
+       r = mkdir(dirname, 0);
+
+END:
+       free(dirname);
+
+       return r;
+}
+
+FILE* pakfire_mktemp(char* path) {
+       pakfire_mkparentdir(path);
+
+       // Create a temporary result file
+       int fd = mkostemp(path, O_CLOEXEC);
+       if (fd < 0)
+               return NULL;
+
+       // Re-open as file handle
+       return fdopen(fd, "w+");
+}