From: Michael Tremer Date: Fri, 12 Mar 2021 18:33:16 +0000 (+0000) Subject: downloader: Use STAILQ for mirrors X-Git-Tag: 0.9.28~1285^2~550 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=53b7430724b1cac5233a1e1e0648ce146cc6b210;p=pakfire.git downloader: Use STAILQ for mirrors Signed-off-by: Michael Tremer --- diff --git a/src/libpakfire/downloader.c b/src/libpakfire/downloader.c index d4add68ba..5c364c3d5 100644 --- a/src/libpakfire/downloader.c +++ b/src/libpakfire/downloader.c @@ -46,8 +46,9 @@ static int curl_initialized = 0; struct pakfire_mirror { + STAILQ_ENTRY(pakfire_mirror) nodes; + char url[PATH_MAX]; - unsigned int priority; unsigned int retries_left; int broken; @@ -65,7 +66,7 @@ struct pakfire_transfer { FILE* f; // Mirrors - int mirror; + struct pakfire_mirror* mirror; }; struct pakfire_downloader { @@ -79,9 +80,8 @@ struct pakfire_downloader { TAILQ_HEAD(transfers, pakfire_transfer) transfers; // Mirror stuff + STAILQ_HEAD(mirrors, pakfire_mirror) mirrors; char baseurl[PATH_MAX]; - struct pakfire_mirror** mirrors; - unsigned int num_mirrors; }; static int pakfire_url_is_absolute(const char* url) { @@ -131,13 +131,10 @@ static int pakfire_downloader_setup_curl(struct pakfire_downloader* downloader) } static void pakfire_downloader_clear_mirrors(struct pakfire_downloader* downloader) { - if (downloader->mirrors) { - for (unsigned int i = 0; i < downloader->num_mirrors; i++) { - free(downloader->mirrors[i]); - } - - free(downloader->mirrors); - downloader->num_mirrors = 0; + while (!STAILQ_EMPTY(&downloader->mirrors)) { + struct pakfire_mirror* mirror = STAILQ_FIRST(&downloader->mirrors); + STAILQ_REMOVE_HEAD(&downloader->mirrors, nodes); + free(mirror); } } @@ -158,6 +155,7 @@ static void pakfire_downloader_free(struct pakfire_downloader* downloader) { while (!TAILQ_EMPTY(&downloader->transfers)) { transfer = TAILQ_LAST(&downloader->transfers, transfers); TAILQ_REMOVE(&downloader->transfers, transfer, nodes); + pakfire_transfer_free(transfer); } // Free mirrors @@ -191,6 +189,9 @@ int pakfire_downloader_create(struct pakfire_downloader** downloader, Pakfire pa // Init transfers queue TAILQ_INIT(&d->transfers); + // Init mirrors + STAILQ_INIT(&d->mirrors); + // Setup cURL int r = pakfire_downloader_setup_curl(d); if (r) @@ -229,26 +230,6 @@ void pakfire_downloader_set_baseurl( snprintf(downloader->baseurl, sizeof(downloader->baseurl) - 1, "%s", baseurl); } -static int pakfire_downloader_mirrors_cmp(const void* ptr1, const void* ptr2) { - struct pakfire_mirror* mirror1 = (struct pakfire_mirror*)ptr1; - struct pakfire_mirror* mirror2 = (struct pakfire_mirror*)ptr2; - - if (mirror1->priority > mirror2->priority) - return 1; - else if (mirror1->priority < mirror2->priority) - return -1; - else - return 0; -} - -static void pakfire_downloader_mirrors_sort(struct pakfire_downloader* downloader) { - if (!downloader->mirrors) - return; - - qsort(downloader->mirrors, downloader->num_mirrors, sizeof(*downloader->mirrors), - pakfire_downloader_mirrors_cmp); -} - static int pakfire_downloader_check_mirrorlist(struct pakfire_downloader* downloader, struct json_object* root) { struct json_object* typeobj = NULL; @@ -325,7 +306,7 @@ int pakfire_downloader_read_mirrorlist(struct pakfire_downloader* downloader, const char* url = json_object_get_string(urlobj); // Add the mirror to the downloader - r = pakfire_downloader_add_mirror(downloader, url, i); + r = pakfire_downloader_add_mirror(downloader, url); if (r) { ERROR(downloader->pakfire, "Could not add mirror %s: %s\n", url, strerror(errno)); @@ -349,28 +330,20 @@ ERROR: } int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader, - const char* url, unsigned int priority) { - // Make space for another mirror - downloader->mirrors = reallocarray(downloader->mirrors, - sizeof(*downloader->mirrors), downloader->num_mirrors + 1); - + const char* url) { // Allocate a new mirror object struct pakfire_mirror* mirror = calloc(1, sizeof(*mirror)); if (!mirror) return ENOMEM; - // Copy attributes + // Copy URL snprintf(mirror->url, sizeof(mirror->url) - 1, "%s", url); - mirror->priority = priority; // Set retries mirror->retries_left = MIRROR_RETRIES; // Append it to the list - downloader->mirrors[downloader->num_mirrors++] = mirror; - - // Sort mirrors - pakfire_downloader_mirrors_sort(downloader); + STAILQ_INSERT_TAIL(&downloader->mirrors, mirror, nodes); DEBUG(downloader->pakfire, "Added mirror %s\n", mirror->url); @@ -410,7 +383,7 @@ int pakfire_downloader_add_transfer( DEBUG(downloader->pakfire, "Adding download of %s\n", url); // Do not allow relative URLs when no mirrors are set - if (!pakfire_url_is_absolute(url) && downloader->num_mirrors == 0) { + if (!pakfire_url_is_absolute(url) && STAILQ_EMPTY(&downloader->mirrors)) { ERROR(downloader->pakfire, "Relative URLs cannot be used without any mirrors set\n"); return EINVAL; } @@ -419,9 +392,6 @@ int pakfire_downloader_add_transfer( if (!transfer) return ENOMEM; - // Select no mirror - transfer->mirror = -1; - // Copy URL snprintf(transfer->url, sizeof(transfer->url) - 1, "%s", url); @@ -478,24 +448,31 @@ ERROR: static int pakfire_transfer_select_mirror(struct pakfire_downloader* downloader, struct pakfire_transfer* transfer) { - for (unsigned int next = transfer->mirror + 1; - next < downloader->num_mirrors; next++) { - struct pakfire_mirror* mirror = downloader->mirrors[next]; + // Choose the next mirror + if (transfer->mirror) + transfer->mirror = STAILQ_NEXT(transfer->mirror, nodes); - // Skip broken mirrors - if (mirror->broken) - continue; + // If no mirror has been selected yet, choose the first one + else + transfer->mirror = STAILQ_FIRST(&downloader->mirrors); - DEBUG(downloader->pakfire, "Selected mirror %s\n", mirror->url); - transfer->mirror = next; + // Skip this mirror if it is broken + while (transfer->mirror && transfer->mirror->broken) { + // Move on to the next mirror + transfer->mirror = STAILQ_NEXT(transfer->mirror, nodes); + } + + // No mirror found + if (!transfer->mirror) { + ERROR(downloader->pakfire, "No mirrors left to try\n"); - return 0; + // No mirrors left + return ENOENT; } - ERROR(downloader->pakfire, "No mirrors left to try\n"); + DEBUG(downloader->pakfire, "Selected mirror %s\n", transfer->mirror->url); - // No mirrors left - return ENOENT; + return 0; } static const char* curl_http_version(long v) { @@ -554,13 +531,12 @@ static int pakfire_transfer_fail(struct pakfire_downloader* downloader, return r; // Did we use a mirror? - if (transfer->mirror >= 0) { - struct pakfire_mirror* mirror = downloader->mirrors[transfer->mirror]; - + if (transfer->mirror) { // Decrease retries and potentially mark mirror as broken - if (!mirror->retries_left--) { - DEBUG(downloader->pakfire, "Consider mirror %s as broken\n", mirror->url); - mirror->broken = 1; + if (!transfer->mirror->retries_left--) { + DEBUG(downloader->pakfire, "Mark mirror %s as broken\n", + transfer->mirror->url); + transfer->mirror->broken = 1; } // Try again with another mirror @@ -678,9 +654,7 @@ static int pakfire_downloader_activate_transfer(struct pakfire_downloader* downl if (r) return r; - struct pakfire_mirror* mirror = downloader->mirrors[transfer->mirror]; - - char* url = pakfire_url_join(mirror->url, transfer->url); + char* url = pakfire_url_join(transfer->mirror->url, transfer->url); if (!url) { ERROR(downloader->pakfire, "Error composing download URL: %s\n", strerror(errno)); diff --git a/src/libpakfire/include/pakfire/downloader.h b/src/libpakfire/include/pakfire/downloader.h index e03f86936..fb7d50627 100644 --- a/src/libpakfire/include/pakfire/downloader.h +++ b/src/libpakfire/include/pakfire/downloader.h @@ -38,7 +38,7 @@ void pakfire_downloader_set_baseurl(struct pakfire_downloader* downloader, const int pakfire_downloader_read_mirrorlist(struct pakfire_downloader* downloader, const char* path); int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader, - const char* url, unsigned int priority); + const char* url); int pakfire_downloader_retrieve(struct pakfire_downloader* downloader, const char* url, const char* path);