From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 12 Mar 2021 17:44:54 +0000 (+0000)
Subject: downloader: Use next mirror if download failed
X-Git-Tag: 0.9.28~1285^2~552
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7da3f0b0d4cf378e0297b26340808de57a8765c5;p=pakfire.git

downloader: Use next mirror if download failed

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/libpakfire/downloader.c b/src/libpakfire/downloader.c
index 60a24d844..61e5dcaf8 100644
--- a/src/libpakfire/downloader.c
+++ b/src/libpakfire/downloader.c
@@ -33,6 +33,9 @@
 #include <pakfire/types.h>
 #include <pakfire/util.h>
 
+// Retry a mirror up to N times before marking it as broken
+#define MIRROR_RETRIES			3
+
 // The number of concurrent downloads
 #define MAX_PARALLEL			4
 
@@ -45,6 +48,9 @@ static int curl_initialized = 0;
 struct pakfire_mirror {
 	char url[PATH_MAX];
 	unsigned int priority;
+
+	unsigned int retries_left;
+	int broken;
 };
 
 struct pakfire_transfer {
@@ -59,7 +65,7 @@ struct pakfire_transfer {
 	FILE* f;
 
 	// Mirrors
-	unsigned int current_mirror;
+	int current_mirror;
 };
 
 struct pakfire_downloader {
@@ -357,6 +363,9 @@ int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader,
 	snprintf(mirror->url, sizeof(mirror->url) - 1, "%s", url);
 	mirror->priority = priority;
 
+	// Set retries
+	mirror->retries_left = MIRROR_RETRIES;
+
 	// Append it to the list
 	downloader->mirrors[downloader->num_mirrors++] = mirror;
 
@@ -410,6 +419,9 @@ int pakfire_downloader_add_transfer(
 	if (!transfer)
 		return ENOMEM;
 
+	// Select no mirror
+	transfer->current_mirror = -1;
+
 	// Copy URL
 	snprintf(transfer->url, sizeof(transfer->url) - 1, "%s", url);
 
@@ -464,6 +476,28 @@ ERROR:
 	return 1;
 }
 
+static int pakfire_transfer_select_mirror(struct pakfire_downloader* downloader,
+		struct pakfire_transfer* transfer) {
+	for (unsigned int next = transfer->current_mirror + 1;
+			next < downloader->num_mirrors; next++) {
+		struct pakfire_mirror* mirror = downloader->mirrors[next];
+
+		// Skip broken mirrors
+		if (mirror->broken)
+			continue;
+
+		DEBUG(downloader->pakfire, "Selected mirror %s\n", mirror->url);
+		transfer->current_mirror = next;
+
+		return 0;
+	}
+
+	ERROR(downloader->pakfire, "No mirrors left to try\n");
+
+	// No mirrors left
+	return ENOENT;
+}
+
 static const char* curl_http_version(long v) {
 	switch (v) {
 		case CURL_HTTP_VERSION_2_0:
@@ -519,6 +553,20 @@ static int pakfire_transfer_fail(struct pakfire_downloader* downloader,
 	if (r)
 		return r;
 
+	// Did we use a mirror?
+	if (transfer->current_mirror >= 0) {
+		struct pakfire_mirror* mirror = downloader->mirrors[transfer->current_mirror];
+
+		// Decrease retries and potentially mark mirror as broken
+		if (!mirror->retries_left--) {
+			DEBUG(downloader->pakfire, "Consider mirror %s as broken\n", mirror->url);
+			mirror->broken = 1;
+		}
+
+		// Try again with another mirror
+		return EAGAIN;
+	}
+
 	return 0;
 }
 
@@ -626,7 +674,10 @@ static int pakfire_downloader_activate_transfer(struct pakfire_downloader* downl
 
 	// Join path if we are using mirrors
 	} else {
-		// XXX for now
+		int r = pakfire_transfer_select_mirror(downloader, transfer);
+		if (r)
+			return r;
+
 		struct pakfire_mirror* mirror = downloader->mirrors[transfer->current_mirror];
 
 		char* url = pakfire_url_join(mirror->url, transfer->url);
@@ -685,16 +736,23 @@ int pakfire_downloader_run(struct pakfire_downloader* downloader) {
 
 				// Update reference to transfer
 				curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &transfer);
+				curl_multi_remove_handle(downloader->curl, transfer->handle);
 
 				// Handle whatever comes after the transfer
 				r = pakfire_transfer_done(downloader, transfer, msg);
-				if (r)
+				if (r) {
+					// We will try another mirror and therefore insert this
+					// transfer to be picked up again next
+					if (r == EAGAIN) {
+						TAILQ_INSERT_TAIL(&downloader->transfers, transfer, nodes);
+						continue;
+					}
+
 					success = r;
+				}
 
-				// Remove and destroy the handle
-				curl_multi_remove_handle(downloader->curl, transfer->handle);
+				// Destroy transfer
 				pakfire_transfer_free(transfer);
-
 			// Log any other messages
 			} else {
 				ERROR(downloader->pakfire, "cURL reported error %d\n", msg->msg);