downloader: Use next mirror if download failed

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/libpakfire/downloader.c b/src/libpakfire/downloader.c
index 60a24d8442fdb98d11f9acf6975502be6290d39b..61e5dcaf894d1748b5530404eba9c794f644b819 100644 (file)
--- a/src/libpakfire/downloader.c
+++ b/src/libpakfire/downloader.c
@@ -33,6 +33,9 @@
 #include <pakfire/types.h>
 #include <pakfire/util.h>
 
+// Retry a mirror up to N times before marking it as broken
+#define MIRROR_RETRIES                 3
+
 // The number of concurrent downloads
 #define MAX_PARALLEL                   4
 
@@ -45,6 +48,9 @@ static int curl_initialized = 0;
 struct pakfire_mirror {
        char url[PATH_MAX];
        unsigned int priority;
+
+       unsigned int retries_left;
+       int broken;
 };
 
 struct pakfire_transfer {
@@ -59,7 +65,7 @@ struct pakfire_transfer {
        FILE* f;
 
        // Mirrors
-       unsigned int current_mirror;
+       int current_mirror;
 };
 
 struct pakfire_downloader {
@@ -357,6 +363,9 @@ int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader,
        snprintf(mirror->url, sizeof(mirror->url) - 1, "%s", url);
        mirror->priority = priority;
 
+       // Set retries
+       mirror->retries_left = MIRROR_RETRIES;
+
        // Append it to the list
        downloader->mirrors[downloader->num_mirrors++] = mirror;
 
@@ -410,6 +419,9 @@ int pakfire_downloader_add_transfer(
        if (!transfer)
                return ENOMEM;
 
+       // Select no mirror
+       transfer->current_mirror = -1;
+
        // Copy URL
        snprintf(transfer->url, sizeof(transfer->url) - 1, "%s", url);
 
@@ -464,6 +476,28 @@ ERROR:
        return 1;
 }
 
+static int pakfire_transfer_select_mirror(struct pakfire_downloader* downloader,
+               struct pakfire_transfer* transfer) {
+       for (unsigned int next = transfer->current_mirror + 1;
+                       next < downloader->num_mirrors; next++) {
+               struct pakfire_mirror* mirror = downloader->mirrors[next];
+
+               // Skip broken mirrors
+               if (mirror->broken)
+                       continue;
+
+               DEBUG(downloader->pakfire, "Selected mirror %s\n", mirror->url);
+               transfer->current_mirror = next;
+
+               return 0;
+       }
+
+       ERROR(downloader->pakfire, "No mirrors left to try\n");
+
+       // No mirrors left
+       return ENOENT;
+}
+
 static const char* curl_http_version(long v) {
        switch (v) {
                case CURL_HTTP_VERSION_2_0:
@@ -519,6 +553,20 @@ static int pakfire_transfer_fail(struct pakfire_downloader* downloader,
        if (r)
                return r;
 
+       // Did we use a mirror?
+       if (transfer->current_mirror >= 0) {
+               struct pakfire_mirror* mirror = downloader->mirrors[transfer->current_mirror];
+
+               // Decrease retries and potentially mark mirror as broken
+               if (!mirror->retries_left--) {
+                       DEBUG(downloader->pakfire, "Consider mirror %s as broken\n", mirror->url);
+                       mirror->broken = 1;
+               }
+
+               // Try again with another mirror
+               return EAGAIN;
+       }
+
        return 0;
 }
 
@@ -626,7 +674,10 @@ static int pakfire_downloader_activate_transfer(struct pakfire_downloader* downl
 
        // Join path if we are using mirrors
        } else {
-               // XXX for now
+               int r = pakfire_transfer_select_mirror(downloader, transfer);
+               if (r)
+                       return r;
+
                struct pakfire_mirror* mirror = downloader->mirrors[transfer->current_mirror];
 
                char* url = pakfire_url_join(mirror->url, transfer->url);
@@ -685,16 +736,23 @@ int pakfire_downloader_run(struct pakfire_downloader* downloader) {
 
                                // Update reference to transfer
                                curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &transfer);
+                               curl_multi_remove_handle(downloader->curl, transfer->handle);
 
                                // Handle whatever comes after the transfer
                                r = pakfire_transfer_done(downloader, transfer, msg);
-                               if (r)
+                               if (r) {
+                                       // We will try another mirror and therefore insert this
+                                       // transfer to be picked up again next
+                                       if (r == EAGAIN) {
+                                               TAILQ_INSERT_TAIL(&downloader->transfers, transfer, nodes);
+                                               continue;
+                                       }
+
                                        success = r;
+                               }
 
-                               // Remove and destroy the handle
-                               curl_multi_remove_handle(downloader->curl, transfer->handle);
+                               // Destroy transfer
                                pakfire_transfer_free(transfer);
-
                        // Log any other messages
                        } else {
                                ERROR(downloader->pakfire, "cURL reported error %d\n", msg->msg);