From: Michael Tremer Date: Wed, 24 Mar 2021 15:43:06 +0000 (+0000) Subject: downloader: Make mirrorlists independent X-Git-Tag: 0.9.28~1285^2~469 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=04bde19fa5270fa312be07e4c551141309bde7cd;p=pakfire.git downloader: Make mirrorlists independent Signed-off-by: Michael Tremer --- diff --git a/src/libpakfire/dist.c b/src/libpakfire/dist.c index 781b2b9fc..1c6be00d6 100644 --- a/src/libpakfire/dist.c +++ b/src/libpakfire/dist.c @@ -124,7 +124,7 @@ static int pakfire_dist_download_source(Pakfire pakfire, const char* filename, pakfire_downloader_set_baseurl(downloader, "https://source.ipfire.org/source-3.x/"); // Add download - r = pakfire_downloader_add_transfer(downloader, filename, cache_path); + r = pakfire_downloader_add_transfer(downloader, NULL, filename, cache_path); if (r) goto ERROR; diff --git a/src/libpakfire/downloader.c b/src/libpakfire/downloader.c index cc8959479..edafd5859 100644 --- a/src/libpakfire/downloader.c +++ b/src/libpakfire/downloader.c @@ -67,6 +67,7 @@ struct pakfire_transfer { FILE* f; // Mirrors + struct pakfire_mirrorlist* mirrors; struct pakfire_mirror* mirror; }; @@ -81,7 +82,6 @@ struct pakfire_downloader { TAILQ_HEAD(transfers, pakfire_transfer) transfers; // Mirror stuff - STAILQ_HEAD(mirrors, pakfire_mirror) mirrors; char baseurl[PATH_MAX]; }; @@ -131,14 +131,6 @@ static int pakfire_downloader_setup_curl(struct pakfire_downloader* downloader) return 0; } -static void pakfire_downloader_clear_mirrors(struct pakfire_downloader* downloader) { - while (!STAILQ_EMPTY(&downloader->mirrors)) { - struct pakfire_mirror* mirror = STAILQ_FIRST(&downloader->mirrors); - STAILQ_REMOVE_HEAD(&downloader->mirrors, nodes); - free(mirror); - } -} - static void pakfire_transfer_free(struct pakfire_transfer* transfer) { if (transfer->handle) curl_easy_cleanup(transfer->handle); @@ -151,6 +143,9 @@ static void pakfire_transfer_free(struct pakfire_transfer* transfer) { if (transfer->f) fclose(transfer->f); + if (transfer->mirrors) + pakfire_mirrorlist_unref(transfer->mirrors); + free(transfer); } @@ -163,9 +158,6 @@ static void pakfire_downloader_free(struct pakfire_downloader* downloader) { pakfire_transfer_free(transfer); } - // Free mirrors - pakfire_downloader_clear_mirrors(downloader); - if (downloader->curl) curl_multi_cleanup(downloader->curl); @@ -194,9 +186,6 @@ int pakfire_downloader_create(struct pakfire_downloader** downloader, Pakfire pa // Init transfers queue TAILQ_INIT(&d->transfers); - // Init mirrors - STAILQ_INIT(&d->mirrors); - // Setup cURL int r = pakfire_downloader_setup_curl(d); if (r) @@ -235,127 +224,6 @@ void pakfire_downloader_set_baseurl( snprintf(downloader->baseurl, sizeof(downloader->baseurl) - 1, "%s", baseurl); } -static int pakfire_downloader_check_mirrorlist(struct pakfire_downloader* downloader, - struct json_object* root) { - struct json_object* typeobj = NULL; - int r = 1; - - r = json_object_object_get_ex(root, "type", &typeobj); - if (r == FALSE) { - ERROR(downloader->pakfire, "mirrorlist does not have a 'type' attribute\n"); - goto ERROR; - } - - const char* type = json_object_get_string(typeobj); - if (!type) { - ERROR(downloader->pakfire, "mirrorlist has an empty or unknown 'type' attribute\n"); - goto ERROR; - } - - if (strcmp(type, "mirrorlist") != 0) { - ERROR(downloader->pakfire, "Unexpected type: %s\n", type); - goto ERROR; - } - - // Success - r = 0; - -ERROR: - if (typeobj) - json_object_put(typeobj); - - return r; -} - -int pakfire_downloader_read_mirrorlist(struct pakfire_downloader* downloader, - const char* path) { - DEBUG(downloader->pakfire, "Reading mirrorlist from %s\n", path); - - struct json_object* json = pakfire_json_parse_from_file(downloader->pakfire, path); - if (!json) { - ERROR(downloader->pakfire, "Could not parse mirrorlist from %s: %s\n", - path, strerror(errno)); - return 1; - } - - struct json_object* mirrors = NULL; - - // Check if we found a valid mirrorlist - int r = pakfire_downloader_check_mirrorlist(downloader, json); - if (r) - goto ERROR; - - // Clear all existing mirrors - pakfire_downloader_clear_mirrors(downloader); - - // Add the new mirrors - r = json_object_object_get_ex(json, "mirrors", &mirrors); - if (r == FALSE) { - DEBUG(downloader->pakfire, "Mirrorlist has no mirrors\n"); - r = 0; - goto ERROR; - } - - size_t num_mirrors = json_object_array_length(mirrors); - - for (unsigned int i = 0; i < num_mirrors; i++) { - struct json_object* mirror = json_object_array_get_idx(mirrors, i); - if (!mirror) - continue; - - // Find URL - struct json_object* urlobj; - r = json_object_object_get_ex(mirror, "url", &urlobj); - if (r == FALSE) - goto NEXT; - - const char* url = json_object_get_string(urlobj); - - // Add the mirror to the downloader - r = pakfire_downloader_add_mirror(downloader, url); - if (r) { - ERROR(downloader->pakfire, "Could not add mirror %s: %s\n", - url, strerror(errno)); - goto NEXT; - } - -NEXT: - json_object_put(mirror); - } - - // Success - r = 0; - -ERROR: - if (mirrors) - json_object_put(mirrors); - if (json) - json_object_put(json); - - return r; -} - -int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader, - const char* url) { - // Allocate a new mirror object - struct pakfire_mirror* mirror = calloc(1, sizeof(*mirror)); - if (!mirror) - return ENOMEM; - - // Copy URL - snprintf(mirror->url, sizeof(mirror->url) - 1, "%s", url); - - // Set retries - mirror->retries_left = MIRROR_RETRIES; - - // Append it to the list - STAILQ_INSERT_TAIL(&downloader->mirrors, mirror, nodes); - - DEBUG(downloader->pakfire, "Added mirror %s\n", mirror->url); - - return 0; -} - #ifdef ENABLE_DEBUG static int debug_callback(CURL *handle, curl_infotype type, char* data, size_t size, void* private) { @@ -384,16 +252,14 @@ static int debug_callback(CURL *handle, curl_infotype type, } #endif -int pakfire_downloader_add_transfer( - struct pakfire_downloader* downloader, const char* url, const char* path) { +int pakfire_downloader_add_transfer(struct pakfire_downloader* downloader, + struct pakfire_mirrorlist* mirrors, const char* url, const char* path) { DEBUG(downloader->pakfire, "Adding download of %s\n", url); // Do not allow relative URLs when no mirrors are set - if (!pakfire_url_is_absolute(url)) { - if (STAILQ_EMPTY(&downloader->mirrors) && !*downloader->baseurl) { - ERROR(downloader->pakfire, "Relative URLs cannot be used without any mirrors set\n"); - return EINVAL; - } + if (!pakfire_url_is_absolute(url) && !mirrors) { + ERROR(downloader->pakfire, "Relative URLs cannot be used without a mirrorlist\n"); + return EINVAL; } struct pakfire_transfer* transfer = calloc(1, sizeof(*transfer)); @@ -409,6 +275,10 @@ int pakfire_downloader_add_transfer( // Make path for the temporary file (must be on the same file system) snprintf(transfer->tempfile, sizeof(transfer->tempfile) - 1, "%s.XXXXXX", path); + // Keep a reference to the mirrorlist + if (mirrors) + transfer->mirrors = pakfire_mirrorlist_ref(mirrors); + // Allocate handle transfer->handle = curl_easy_init(); if (!transfer->handle) @@ -462,7 +332,7 @@ static int pakfire_transfer_select_mirror(struct pakfire_downloader* downloader, // If no mirror has been selected yet, choose the first one else - transfer->mirror = STAILQ_FIRST(&downloader->mirrors); + transfer->mirror = pakfire_mirrorlist_first(transfer->mirrors); // Skip this mirror if it is broken while (transfer->mirror && transfer->mirror->broken) { @@ -639,13 +509,13 @@ static int pakfire_transfer_done(struct pakfire_downloader* downloader, } int pakfire_downloader_retrieve(struct pakfire_downloader* downloader, - const char* url, const char* path) { + struct pakfire_mirrorlist* mirrors, const char* url, const char* path) { // Do not run this when there are transfers pending if (!TAILQ_EMPTY(&downloader->transfers)) return EBUSY; // Add the transfer - int r = pakfire_downloader_add_transfer(downloader, url, path); + int r = pakfire_downloader_add_transfer(downloader, mirrors, url, path); if (r) return r; @@ -663,7 +533,7 @@ static int pakfire_downloader_activate_transfer(struct pakfire_downloader* downl curl_easy_setopt(transfer->handle, CURLOPT_URL, transfer->url); // Join path if we are using mirrors - } else if (!STAILQ_EMPTY(&downloader->mirrors)) { + } else if (!pakfire_mirrorlist_empty(transfer->mirrors)) { int r = pakfire_transfer_select_mirror(downloader, transfer); if (r) return r; @@ -772,3 +642,183 @@ int pakfire_downloader_run(struct pakfire_downloader* downloader) { // Success return success; } + +struct pakfire_mirrorlist { + Pakfire pakfire; + int nrefs; + + STAILQ_HEAD(mirrors, pakfire_mirror) mirrors; +}; + +static void pakfire_mirrorlist_clear(struct pakfire_mirrorlist* ml) { + while (!STAILQ_EMPTY(&ml->mirrors)) { + struct pakfire_mirror* mirror = STAILQ_FIRST(&ml->mirrors); + STAILQ_REMOVE_HEAD(&ml->mirrors, nodes); + free(mirror); + } +} + +static void pakfire_mirrorlist_free(struct pakfire_mirrorlist* ml) { + pakfire_mirrorlist_clear(ml); + + pakfire_unref(ml->pakfire); + free(ml); +} + +int pakfire_mirrorlist_create(struct pakfire_mirrorlist** mirrorlist, Pakfire pakfire) { + struct pakfire_mirrorlist* ml = calloc(1, sizeof(*ml)); + if (!ml) + return ENOMEM; + + ml->pakfire = pakfire_ref(pakfire); + ml->nrefs = 1; + + // Init mirrors + STAILQ_INIT(&ml->mirrors); + + *mirrorlist = ml; + + return 0; +} + +struct pakfire_mirrorlist* pakfire_mirrorlist_ref(struct pakfire_mirrorlist* ml) { + ++ml->nrefs; + + return ml; +} + +struct pakfire_mirrorlist* pakfire_mirrorlist_unref(struct pakfire_mirrorlist* ml) { + if (--ml->nrefs > 0) + return ml; + + pakfire_mirrorlist_free(ml); + + return NULL; +} + +static int pakfire_mirrorlist_check_mirrorlist(struct pakfire_mirrorlist* ml, + struct json_object* root) { + struct json_object* typeobj = NULL; + int r = 1; + + r = json_object_object_get_ex(root, "type", &typeobj); + if (r == FALSE) { + ERROR(ml->pakfire, "mirrorlist does not have a 'type' attribute\n"); + goto ERROR; + } + + const char* type = json_object_get_string(typeobj); + if (!type) { + ERROR(ml->pakfire, "mirrorlist has an empty or unknown 'type' attribute\n"); + goto ERROR; + } + + if (strcmp(type, "mirrorlist") != 0) { + ERROR(ml->pakfire, "Unexpected type: %s\n", type); + goto ERROR; + } + + // Success + r = 0; + +ERROR: + if (typeobj) + json_object_put(typeobj); + + return r; +} + +int pakfire_mirrorlist_read(struct pakfire_mirrorlist* ml, const char* path) { + DEBUG(ml->pakfire, "Reading mirrorlist from %s\n", path); + + struct json_object* json = pakfire_json_parse_from_file(ml->pakfire, path); + if (!json) { + ERROR(ml->pakfire, "Could not parse mirrorlist from %s: %s\n", + path, strerror(errno)); + return 1; + } + + struct json_object* mirrors = NULL; + + // Check if we found a valid mirrorlist + int r = pakfire_mirrorlist_check_mirrorlist(ml, json); + if (r) + goto ERROR; + + // Clear all existing mirrors + pakfire_mirrorlist_clear(ml); + + // Add the new mirrors + r = json_object_object_get_ex(json, "mirrors", &mirrors); + if (r == FALSE) { + DEBUG(ml->pakfire, "Mirrorlist has no mirrors\n"); + r = 0; + goto ERROR; + } + + size_t num_mirrors = json_object_array_length(mirrors); + + for (unsigned int i = 0; i < num_mirrors; i++) { + struct json_object* mirror = json_object_array_get_idx(mirrors, i); + if (!mirror) + continue; + + // Find URL + struct json_object* urlobj; + r = json_object_object_get_ex(mirror, "url", &urlobj); + if (r == FALSE) + goto NEXT; + + const char* url = json_object_get_string(urlobj); + + // Add the mirror to the downloader + r = pakfire_mirrorlist_add_mirror(ml, url); + if (r) { + ERROR(ml->pakfire, "Could not add mirror %s: %s\n", + url, strerror(errno)); + goto NEXT; + } + +NEXT: + json_object_put(mirror); + } + + // Success + r = 0; + +ERROR: + if (mirrors) + json_object_put(mirrors); + if (json) + json_object_put(json); + + return r; +} + +int pakfire_mirrorlist_add_mirror(struct pakfire_mirrorlist* ml, const char* url) { + // Allocate a new mirror object + struct pakfire_mirror* mirror = calloc(1, sizeof(*mirror)); + if (!mirror) + return ENOMEM; + + // Copy URL + snprintf(mirror->url, sizeof(mirror->url) - 1, "%s", url); + + // Set retries + mirror->retries_left = MIRROR_RETRIES; + + // Append it to the list + STAILQ_INSERT_TAIL(&ml->mirrors, mirror, nodes); + + DEBUG(ml->pakfire, "Added mirror %s\n", mirror->url); + + return 0; +} + +int pakfire_mirrorlist_empty(struct pakfire_mirrorlist* ml) { + return STAILQ_EMPTY(&ml->mirrors); +} + +struct pakfire_mirror* pakfire_mirrorlist_first(struct pakfire_mirrorlist* ml) { + return STAILQ_FIRST(&ml->mirrors); +} diff --git a/src/libpakfire/include/pakfire/downloader.h b/src/libpakfire/include/pakfire/downloader.h index fb7d50627..0f5bfa63e 100644 --- a/src/libpakfire/include/pakfire/downloader.h +++ b/src/libpakfire/include/pakfire/downloader.h @@ -26,6 +26,7 @@ #include struct pakfire_downloader; +struct pakfire_mirrorlist; int pakfire_downloader_create(struct pakfire_downloader** downloader, Pakfire pakfire); @@ -35,18 +36,26 @@ struct pakfire_downloader* pakfire_downloader_unref(struct pakfire_downloader* d const char* pakfire_downloader_get_baseurl(struct pakfire_downloader* downloader); void pakfire_downloader_set_baseurl(struct pakfire_downloader* downloader, const char* baseurl); -int pakfire_downloader_read_mirrorlist(struct pakfire_downloader* downloader, - const char* path); -int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader, - const char* url); - int pakfire_downloader_retrieve(struct pakfire_downloader* downloader, - const char* url, const char* path); + struct pakfire_mirrorlist* mirrors, const char* url, const char* path); int pakfire_downloader_add_transfer(struct pakfire_downloader* downloader, - const char* url, const char* path); + struct pakfire_mirrorlist* mirrors, const char* url, const char* path); int pakfire_downloader_run(struct pakfire_downloader* downloader); +// Mirror lists + +int pakfire_mirrorlist_create(struct pakfire_mirrorlist** ml, Pakfire pakfire); + +struct pakfire_mirrorlist* pakfire_mirrorlist_ref(struct pakfire_mirrorlist* ml); +struct pakfire_mirrorlist* pakfire_mirrorlist_unref(struct pakfire_mirrorlist* ml); + +int pakfire_mirrorlist_read(struct pakfire_mirrorlist* ml, const char* path); +int pakfire_mirrorlist_add_mirror(struct pakfire_mirrorlist* ml, const char* url); + +int pakfire_mirrorlist_empty(struct pakfire_mirrorlist* ml); +struct pakfire_mirror* pakfire_mirrorlist_first(struct pakfire_mirrorlist* ml); + #endif #endif /* PAKFIRE_DOWNLOADER_H */ diff --git a/src/libpakfire/repo.c b/src/libpakfire/repo.c index 7bb4dd3c9..10446df7d 100644 --- a/src/libpakfire/repo.c +++ b/src/libpakfire/repo.c @@ -63,6 +63,7 @@ struct pakfire_repo_appdata { char metadata[PATH_MAX]; // Mirrorlist + struct pakfire_mirrorlist* mirrors; char* mirrorlist_url; char mirrorlist[PATH_MAX]; }; @@ -92,7 +93,7 @@ static struct pakfire_downloader* pakfire_repo_downloader(PakfireRepo repo) { // Load all mirrors (if the mirrorlist exists) if (repo->appdata->mirrorlist && pakfire_path_exists(repo->appdata->mirrorlist)) { - r = pakfire_downloader_read_mirrorlist(repo->downloader, repo->appdata->mirrorlist); + r = pakfire_mirrorlist_read(repo->appdata->mirrors, repo->appdata->mirrorlist); if (r) return NULL; } @@ -120,7 +121,8 @@ static int pakfire_repo_download_database(PakfireRepo repo, const char* database return 1; // Retrieve the database file - int r = pakfire_downloader_retrieve(downloader, database_url, cache_path); + int r = pakfire_downloader_retrieve(downloader, + repo->appdata->mirrors, database_url, cache_path); pakfire_downloader_unref(downloader); return r; @@ -214,12 +216,12 @@ static int pakfire_repo_refresh_mirrorlist(PakfireRepo repo, const int force) { // Try to retrieve the mirrorlist r = pakfire_downloader_retrieve(downloader, - repo->appdata->mirrorlist_url, repo->appdata->mirrorlist); + NULL, repo->appdata->mirrorlist_url, repo->appdata->mirrorlist); if (r) goto ERROR; // Parse it - r = pakfire_downloader_read_mirrorlist(downloader, repo->appdata->mirrorlist); + r = pakfire_mirrorlist_read(repo->appdata->mirrors, repo->appdata->mirrorlist); if (r) { unlink(repo->appdata->mirrorlist); goto ERROR; @@ -252,7 +254,7 @@ static int pakfire_repo_refresh_metadata(PakfireRepo repo, const int force) { // Try to download the metadata int r = pakfire_downloader_retrieve(downloader, - "repodata/repomd.json", repo->appdata->metadata); + repo->appdata->mirrors, "repodata/repomd.json", repo->appdata->metadata); if (r) goto ERROR; @@ -284,6 +286,9 @@ static void free_repo_appdata(struct pakfire_repo_appdata* appdata) { if (appdata->mirrorlist_url) free(appdata->mirrorlist_url); + if (appdata->mirrors) + pakfire_mirrorlist_unref(appdata->mirrors); + free(appdata); } @@ -351,6 +356,11 @@ PAKFIRE_EXPORT PakfireRepo pakfire_repo_create(Pakfire pakfire, const char* name goto ERROR; } + // Allocate mirrorlist + r = pakfire_mirrorlist_create(&repo->appdata->mirrors, repo->pakfire); + if (r) + goto ERROR; + // Setup/clear repository data r = pakfire_repo_clear(repo); if (r) @@ -843,7 +853,7 @@ int pakfire_repo_download_packages(PakfireRepo repo, PakfirePackageList packages // Add transfer to downloader r = pakfire_downloader_add_transfer(downloader, - pakfire_package_get_filename(pkg), cache_path); + repo->appdata->mirrors, pakfire_package_get_filename(pkg), cache_path); if (r) { pakfire_package_unref(pkg); free(cache_path); diff --git a/tests/libpakfire/downloader.c b/tests/libpakfire/downloader.c index 53f19efd4..a5353dcd8 100644 --- a/tests/libpakfire/downloader.c +++ b/tests/libpakfire/downloader.c @@ -36,7 +36,7 @@ static int test_simple(const struct test* t) { ASSERT(r == 0); // Retrieve a file - r = pakfire_downloader_retrieve(d, DOWNLOAD_URL, DOWNLOAD_PATH); + r = pakfire_downloader_retrieve(d, NULL, DOWNLOAD_URL, DOWNLOAD_PATH); ASSERT(r == 0); // Cleanup @@ -53,11 +53,11 @@ static int test_retrieve_with_pending_transfers(const struct test* t) { ASSERT(r == 0); // Add a transfer - r = pakfire_downloader_add_transfer(d, DOWNLOAD_URL, DOWNLOAD_PATH); + r = pakfire_downloader_add_transfer(d, NULL, DOWNLOAD_URL, DOWNLOAD_PATH); ASSERT(r == 0); // Retrieve a file - r = pakfire_downloader_retrieve(d, DOWNLOAD_URL, DOWNLOAD_PATH); + r = pakfire_downloader_retrieve(d, NULL, DOWNLOAD_URL, DOWNLOAD_PATH); ASSERT(r == EBUSY); // Cleanup