downloader: Make mirrorlists independent

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/libpakfire/dist.c b/src/libpakfire/dist.c
index 781b2b9fcae80cde7907baeb9d00629afb9e0064..1c6be00d6134dff7a68f93cf2ca06a54ac5b104f 100644 (file)
--- a/src/libpakfire/dist.c
+++ b/src/libpakfire/dist.c
@@ -124,7 +124,7 @@ static int pakfire_dist_download_source(Pakfire pakfire, const char* filename,
        pakfire_downloader_set_baseurl(downloader, "https://source.ipfire.org/source-3.x/");
 
        // Add download
-       r = pakfire_downloader_add_transfer(downloader, filename, cache_path);
+       r = pakfire_downloader_add_transfer(downloader, NULL, filename, cache_path);
        if (r)
                goto ERROR;
 

diff --git a/src/libpakfire/downloader.c b/src/libpakfire/downloader.c
index cc8959479ac9c7712b9ed53eec7eccfbc1b3478b..edafd5859b7415bad1031baf93ca304995f07dfd 100644 (file)
--- a/src/libpakfire/downloader.c
+++ b/src/libpakfire/downloader.c
@@ -67,6 +67,7 @@ struct pakfire_transfer {
        FILE* f;
 
        // Mirrors
+       struct pakfire_mirrorlist* mirrors;
        struct pakfire_mirror* mirror;
 };
 
@@ -81,7 +82,6 @@ struct pakfire_downloader {
        TAILQ_HEAD(transfers, pakfire_transfer) transfers;
 
        // Mirror stuff
-       STAILQ_HEAD(mirrors, pakfire_mirror) mirrors;
        char baseurl[PATH_MAX];
 };
 
@@ -131,14 +131,6 @@ static int pakfire_downloader_setup_curl(struct pakfire_downloader* downloader)
        return 0;
 }
 
-static void pakfire_downloader_clear_mirrors(struct pakfire_downloader* downloader) {
-       while (!STAILQ_EMPTY(&downloader->mirrors)) {
-               struct pakfire_mirror* mirror = STAILQ_FIRST(&downloader->mirrors);
-               STAILQ_REMOVE_HEAD(&downloader->mirrors, nodes);
-               free(mirror);
-       }
-}
-
 static void pakfire_transfer_free(struct pakfire_transfer* transfer) {
        if (transfer->handle)
                curl_easy_cleanup(transfer->handle);
@@ -151,6 +143,9 @@ static void pakfire_transfer_free(struct pakfire_transfer* transfer) {
        if (transfer->f)
                fclose(transfer->f);
 
+       if (transfer->mirrors)
+               pakfire_mirrorlist_unref(transfer->mirrors);
+
        free(transfer);
 }
 
@@ -163,9 +158,6 @@ static void pakfire_downloader_free(struct pakfire_downloader* downloader) {
                pakfire_transfer_free(transfer);
        }
 
-       // Free mirrors
-       pakfire_downloader_clear_mirrors(downloader);
-
        if (downloader->curl)
                curl_multi_cleanup(downloader->curl);
 
@@ -194,9 +186,6 @@ int pakfire_downloader_create(struct pakfire_downloader** downloader, Pakfire pa
        // Init transfers queue
        TAILQ_INIT(&d->transfers);
 
-       // Init mirrors
-       STAILQ_INIT(&d->mirrors);
-
        // Setup cURL
        int r = pakfire_downloader_setup_curl(d);
        if (r)
@@ -235,127 +224,6 @@ void pakfire_downloader_set_baseurl(
        snprintf(downloader->baseurl, sizeof(downloader->baseurl) - 1, "%s", baseurl);
 }
 
-static int pakfire_downloader_check_mirrorlist(struct pakfire_downloader* downloader,
-               struct json_object* root) {
-       struct json_object* typeobj = NULL;
-       int r = 1;
-
-       r = json_object_object_get_ex(root, "type", &typeobj);
-       if (r == FALSE) {
-               ERROR(downloader->pakfire, "mirrorlist does not have a 'type' attribute\n");
-               goto ERROR;
-       }
-
-       const char* type = json_object_get_string(typeobj);
-       if (!type) {
-               ERROR(downloader->pakfire, "mirrorlist has an empty or unknown 'type' attribute\n");
-               goto ERROR;
-       }
-
-       if (strcmp(type, "mirrorlist") != 0) {
-               ERROR(downloader->pakfire, "Unexpected type: %s\n", type);
-               goto ERROR;
-       }
-
-       // Success
-       r = 0;
-
-ERROR:
-       if (typeobj)
-               json_object_put(typeobj);
-
-       return r;
-}
-
-int pakfire_downloader_read_mirrorlist(struct pakfire_downloader* downloader,
-               const char* path) {
-       DEBUG(downloader->pakfire, "Reading mirrorlist from %s\n", path);
-
-       struct json_object* json = pakfire_json_parse_from_file(downloader->pakfire, path);
-       if (!json) {
-               ERROR(downloader->pakfire, "Could not parse mirrorlist from %s: %s\n",
-                       path, strerror(errno));
-               return 1;
-       }
-
-       struct json_object* mirrors = NULL;
-
-       // Check if we found a valid mirrorlist
-       int r = pakfire_downloader_check_mirrorlist(downloader, json);
-       if (r)
-               goto ERROR;
-
-       // Clear all existing mirrors
-       pakfire_downloader_clear_mirrors(downloader);
-
-       // Add the new mirrors
-       r = json_object_object_get_ex(json, "mirrors", &mirrors);
-       if (r == FALSE) {
-               DEBUG(downloader->pakfire, "Mirrorlist has no mirrors\n");
-               r = 0;
-               goto ERROR;
-       }
-
-       size_t num_mirrors = json_object_array_length(mirrors);
-
-       for (unsigned int i = 0; i < num_mirrors; i++) {
-               struct json_object* mirror = json_object_array_get_idx(mirrors, i);
-               if (!mirror)
-                       continue;
-
-               // Find URL
-               struct json_object* urlobj;
-               r = json_object_object_get_ex(mirror, "url", &urlobj);
-               if (r == FALSE)
-                       goto NEXT;
-
-               const char* url = json_object_get_string(urlobj);
-
-               // Add the mirror to the downloader
-               r = pakfire_downloader_add_mirror(downloader, url);
-               if (r) {
-                       ERROR(downloader->pakfire, "Could not add mirror %s: %s\n",
-                               url, strerror(errno));
-                       goto NEXT;
-               }
-
-NEXT:
-               json_object_put(mirror);
-       }
-
-       // Success
-       r = 0;
-
-ERROR:
-       if (mirrors)
-               json_object_put(mirrors);
-       if (json)
-               json_object_put(json);
-
-       return r;
-}
-
-int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader,
-               const char* url) {
-       // Allocate a new mirror object
-       struct pakfire_mirror* mirror = calloc(1, sizeof(*mirror));
-       if (!mirror)
-               return ENOMEM;
-
-       // Copy URL
-       snprintf(mirror->url, sizeof(mirror->url) - 1, "%s", url);
-
-       // Set retries
-       mirror->retries_left = MIRROR_RETRIES;
-
-       // Append it to the list
-       STAILQ_INSERT_TAIL(&downloader->mirrors, mirror, nodes);
-
-       DEBUG(downloader->pakfire, "Added mirror %s\n", mirror->url);
-
-       return 0;
-}
-
 #ifdef ENABLE_DEBUG
 static int debug_callback(CURL *handle, curl_infotype type,
                char* data, size_t size, void* private) {
@@ -384,16 +252,14 @@ static int debug_callback(CURL *handle, curl_infotype type,
 }
 #endif
 
-int pakfire_downloader_add_transfer(
-               struct pakfire_downloader* downloader, const char* url, const char* path) {
+int pakfire_downloader_add_transfer(struct pakfire_downloader* downloader,
+               struct pakfire_mirrorlist* mirrors, const char* url, const char* path) {
        DEBUG(downloader->pakfire, "Adding download of %s\n", url);
 
        // Do not allow relative URLs when no mirrors are set
-       if (!pakfire_url_is_absolute(url)) {
-               if (STAILQ_EMPTY(&downloader->mirrors) && !*downloader->baseurl) {
-                       ERROR(downloader->pakfire, "Relative URLs cannot be used without any mirrors set\n");
-                       return EINVAL;
-               }
+       if (!pakfire_url_is_absolute(url) && !mirrors) {
+               ERROR(downloader->pakfire, "Relative URLs cannot be used without a mirrorlist\n");
+               return EINVAL;
        }
 
        struct pakfire_transfer* transfer = calloc(1, sizeof(*transfer));
@@ -409,6 +275,10 @@ int pakfire_downloader_add_transfer(
        // Make path for the temporary file (must be on the same file system)
        snprintf(transfer->tempfile, sizeof(transfer->tempfile) - 1, "%s.XXXXXX", path);
 
+       // Keep a reference to the mirrorlist
+       if (mirrors)
+               transfer->mirrors = pakfire_mirrorlist_ref(mirrors);
+
        // Allocate handle
        transfer->handle = curl_easy_init();
        if (!transfer->handle)
@@ -462,7 +332,7 @@ static int pakfire_transfer_select_mirror(struct pakfire_downloader* downloader,
 
        // If no mirror has been selected yet, choose the first one
        else
-               transfer->mirror = STAILQ_FIRST(&downloader->mirrors);
+               transfer->mirror = pakfire_mirrorlist_first(transfer->mirrors);
 
        // Skip this mirror if it is broken
        while (transfer->mirror && transfer->mirror->broken) {
@@ -639,13 +509,13 @@ static int pakfire_transfer_done(struct pakfire_downloader* downloader,
 }
 
 int pakfire_downloader_retrieve(struct pakfire_downloader* downloader,
-               const char* url, const char* path) {
+               struct pakfire_mirrorlist* mirrors, const char* url, const char* path) {
        // Do not run this when there are transfers pending
        if (!TAILQ_EMPTY(&downloader->transfers))
                return EBUSY;
 
        // Add the transfer
-       int r = pakfire_downloader_add_transfer(downloader, url, path);
+       int r = pakfire_downloader_add_transfer(downloader, mirrors, url, path);
        if (r)
                return r;
 
@@ -663,7 +533,7 @@ static int pakfire_downloader_activate_transfer(struct pakfire_downloader* downl
                curl_easy_setopt(transfer->handle, CURLOPT_URL, transfer->url);
 
        // Join path if we are using mirrors
-       } else if (!STAILQ_EMPTY(&downloader->mirrors)) {
+       } else if (!pakfire_mirrorlist_empty(transfer->mirrors)) {
                int r = pakfire_transfer_select_mirror(downloader, transfer);
                if (r)
                        return r;
@@ -772,3 +642,183 @@ int pakfire_downloader_run(struct pakfire_downloader* downloader) {
        // Success
        return success;
 }
+
+struct pakfire_mirrorlist {
+       Pakfire pakfire;
+       int nrefs;
+
+       STAILQ_HEAD(mirrors, pakfire_mirror) mirrors;
+};
+
+static void pakfire_mirrorlist_clear(struct pakfire_mirrorlist* ml) {
+       while (!STAILQ_EMPTY(&ml->mirrors)) {
+               struct pakfire_mirror* mirror = STAILQ_FIRST(&ml->mirrors);
+               STAILQ_REMOVE_HEAD(&ml->mirrors, nodes);
+               free(mirror);
+       }
+}
+
+static void pakfire_mirrorlist_free(struct pakfire_mirrorlist* ml) {
+       pakfire_mirrorlist_clear(ml);
+
+       pakfire_unref(ml->pakfire);
+       free(ml);
+}
+
+int pakfire_mirrorlist_create(struct pakfire_mirrorlist** mirrorlist, Pakfire pakfire) {
+       struct pakfire_mirrorlist* ml = calloc(1, sizeof(*ml));
+       if (!ml)
+               return ENOMEM;
+
+       ml->pakfire = pakfire_ref(pakfire);
+       ml->nrefs = 1;
+
+       // Init mirrors
+       STAILQ_INIT(&ml->mirrors);
+
+       *mirrorlist = ml;
+
+       return 0;
+}
+
+struct pakfire_mirrorlist* pakfire_mirrorlist_ref(struct pakfire_mirrorlist* ml) {
+       ++ml->nrefs;
+
+       return ml;
+}
+
+struct pakfire_mirrorlist* pakfire_mirrorlist_unref(struct pakfire_mirrorlist* ml) {
+       if (--ml->nrefs > 0)
+               return ml;
+
+       pakfire_mirrorlist_free(ml);
+
+       return NULL;
+}
+
+static int pakfire_mirrorlist_check_mirrorlist(struct pakfire_mirrorlist* ml,
+               struct json_object* root) {
+       struct json_object* typeobj = NULL;
+       int r = 1;
+
+       r = json_object_object_get_ex(root, "type", &typeobj);
+       if (r == FALSE) {
+               ERROR(ml->pakfire, "mirrorlist does not have a 'type' attribute\n");
+               goto ERROR;
+       }
+
+       const char* type = json_object_get_string(typeobj);
+       if (!type) {
+               ERROR(ml->pakfire, "mirrorlist has an empty or unknown 'type' attribute\n");
+               goto ERROR;
+       }
+
+       if (strcmp(type, "mirrorlist") != 0) {
+               ERROR(ml->pakfire, "Unexpected type: %s\n", type);
+               goto ERROR;
+       }
+
+       // Success
+       r = 0;
+
+ERROR:
+       if (typeobj)
+               json_object_put(typeobj);
+
+       return r;
+}
+
+int pakfire_mirrorlist_read(struct pakfire_mirrorlist* ml, const char* path) {
+       DEBUG(ml->pakfire, "Reading mirrorlist from %s\n", path);
+
+       struct json_object* json = pakfire_json_parse_from_file(ml->pakfire, path);
+       if (!json) {
+               ERROR(ml->pakfire, "Could not parse mirrorlist from %s: %s\n",
+                       path, strerror(errno));
+               return 1;
+       }
+
+       struct json_object* mirrors = NULL;
+
+       // Check if we found a valid mirrorlist
+       int r = pakfire_mirrorlist_check_mirrorlist(ml, json);
+       if (r)
+               goto ERROR;
+
+       // Clear all existing mirrors
+       pakfire_mirrorlist_clear(ml);
+
+       // Add the new mirrors
+       r = json_object_object_get_ex(json, "mirrors", &mirrors);
+       if (r == FALSE) {
+               DEBUG(ml->pakfire, "Mirrorlist has no mirrors\n");
+               r = 0;
+               goto ERROR;
+       }
+
+       size_t num_mirrors = json_object_array_length(mirrors);
+
+       for (unsigned int i = 0; i < num_mirrors; i++) {
+               struct json_object* mirror = json_object_array_get_idx(mirrors, i);
+               if (!mirror)
+                       continue;
+
+               // Find URL
+               struct json_object* urlobj;
+               r = json_object_object_get_ex(mirror, "url", &urlobj);
+               if (r == FALSE)
+                       goto NEXT;
+
+               const char* url = json_object_get_string(urlobj);
+
+               // Add the mirror to the downloader
+               r = pakfire_mirrorlist_add_mirror(ml, url);
+               if (r) {
+                       ERROR(ml->pakfire, "Could not add mirror %s: %s\n",
+                               url, strerror(errno));
+                       goto NEXT;
+               }
+
+NEXT:
+               json_object_put(mirror);
+       }
+
+       // Success
+       r = 0;
+
+ERROR:
+       if (mirrors)
+               json_object_put(mirrors);
+       if (json)
+               json_object_put(json);
+
+       return r;
+}
+
+int pakfire_mirrorlist_add_mirror(struct pakfire_mirrorlist* ml, const char* url) {
+       // Allocate a new mirror object
+       struct pakfire_mirror* mirror = calloc(1, sizeof(*mirror));
+       if (!mirror)
+               return ENOMEM;
+
+       // Copy URL
+       snprintf(mirror->url, sizeof(mirror->url) - 1, "%s", url);
+
+       // Set retries
+       mirror->retries_left = MIRROR_RETRIES;
+
+       // Append it to the list
+       STAILQ_INSERT_TAIL(&ml->mirrors, mirror, nodes);
+
+       DEBUG(ml->pakfire, "Added mirror %s\n", mirror->url);
+
+       return 0;
+}
+
+int pakfire_mirrorlist_empty(struct pakfire_mirrorlist* ml) {
+       return STAILQ_EMPTY(&ml->mirrors);
+}
+
+struct pakfire_mirror* pakfire_mirrorlist_first(struct pakfire_mirrorlist* ml) {
+       return STAILQ_FIRST(&ml->mirrors);
+}

diff --git a/src/libpakfire/include/pakfire/downloader.h b/src/libpakfire/include/pakfire/downloader.h
index fb7d50627a9d7a2508114c84a17b637556f91b4d..0f5bfa63eca8bd47cbb6772160b2bbd2a4928ba2 100644 (file)
--- a/src/libpakfire/include/pakfire/downloader.h
+++ b/src/libpakfire/include/pakfire/downloader.h
@@ -26,6 +26,7 @@
 #include <pakfire/types.h>
 
 struct pakfire_downloader;
+struct pakfire_mirrorlist;
 
 int pakfire_downloader_create(struct pakfire_downloader** downloader, Pakfire pakfire);
 
@@ -35,18 +36,26 @@ struct pakfire_downloader* pakfire_downloader_unref(struct pakfire_downloader* d
 const char* pakfire_downloader_get_baseurl(struct pakfire_downloader* downloader);
 void pakfire_downloader_set_baseurl(struct pakfire_downloader* downloader, const char* baseurl);
 
-int pakfire_downloader_read_mirrorlist(struct pakfire_downloader* downloader,
-       const char* path);
-int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader,
-       const char* url);
-
 int pakfire_downloader_retrieve(struct pakfire_downloader* downloader,
-               const char* url, const char* path);
+       struct pakfire_mirrorlist* mirrors, const char* url, const char* path);
 int pakfire_downloader_add_transfer(struct pakfire_downloader* downloader,
-       const char* url, const char* path);
+       struct pakfire_mirrorlist* mirrors, const char* url, const char* path);
 
 int pakfire_downloader_run(struct pakfire_downloader* downloader);
 
+// Mirror lists
+
+int pakfire_mirrorlist_create(struct pakfire_mirrorlist** ml, Pakfire pakfire);
+
+struct pakfire_mirrorlist* pakfire_mirrorlist_ref(struct pakfire_mirrorlist* ml);
+struct pakfire_mirrorlist* pakfire_mirrorlist_unref(struct pakfire_mirrorlist* ml);
+
+int pakfire_mirrorlist_read(struct pakfire_mirrorlist* ml, const char* path);
+int pakfire_mirrorlist_add_mirror(struct pakfire_mirrorlist* ml, const char* url);
+
+int pakfire_mirrorlist_empty(struct pakfire_mirrorlist* ml);
+struct pakfire_mirror* pakfire_mirrorlist_first(struct pakfire_mirrorlist* ml);
+
 #endif
 
 #endif /* PAKFIRE_DOWNLOADER_H */

diff --git a/src/libpakfire/repo.c b/src/libpakfire/repo.c
index 7bb4dd3c95b4a84eeecc3a19d23fda95b02e5f65..10446df7d5a751ca7642bff4fa53940e5a3afe8c 100644 (file)
--- a/src/libpakfire/repo.c
+++ b/src/libpakfire/repo.c
@@ -63,6 +63,7 @@ struct pakfire_repo_appdata {
        char metadata[PATH_MAX];
 
        // Mirrorlist
+       struct pakfire_mirrorlist* mirrors;
        char* mirrorlist_url;
        char mirrorlist[PATH_MAX];
 };
@@ -92,7 +93,7 @@ static struct pakfire_downloader* pakfire_repo_downloader(PakfireRepo repo) {
 
                // Load all mirrors (if the mirrorlist exists)
                if (repo->appdata->mirrorlist && pakfire_path_exists(repo->appdata->mirrorlist)) {
-                       r = pakfire_downloader_read_mirrorlist(repo->downloader, repo->appdata->mirrorlist);
+                       r = pakfire_mirrorlist_read(repo->appdata->mirrors, repo->appdata->mirrorlist);
                        if (r)
                                return NULL;
                }
@@ -120,7 +121,8 @@ static int pakfire_repo_download_database(PakfireRepo repo, const char* database
                return 1;
 
        // Retrieve the database file
-       int r = pakfire_downloader_retrieve(downloader, database_url, cache_path);
+       int r = pakfire_downloader_retrieve(downloader,
+               repo->appdata->mirrors, database_url, cache_path);
        pakfire_downloader_unref(downloader);
 
        return r;
@@ -214,12 +216,12 @@ static int pakfire_repo_refresh_mirrorlist(PakfireRepo repo, const int force) {
 
        // Try to retrieve the mirrorlist
        r = pakfire_downloader_retrieve(downloader,
-                       repo->appdata->mirrorlist_url, repo->appdata->mirrorlist);
+                       NULL, repo->appdata->mirrorlist_url, repo->appdata->mirrorlist);
        if (r)
                goto ERROR;
 
        // Parse it
-       r = pakfire_downloader_read_mirrorlist(downloader, repo->appdata->mirrorlist);
+       r = pakfire_mirrorlist_read(repo->appdata->mirrors, repo->appdata->mirrorlist);
        if (r) {
                unlink(repo->appdata->mirrorlist);
                goto ERROR;
@@ -252,7 +254,7 @@ static int pakfire_repo_refresh_metadata(PakfireRepo repo, const int force) {
 
        // Try to download the metadata
        int r = pakfire_downloader_retrieve(downloader,
-                       "repodata/repomd.json", repo->appdata->metadata);
+                       repo->appdata->mirrors, "repodata/repomd.json", repo->appdata->metadata);
        if (r)
                goto ERROR;
 
@@ -284,6 +286,9 @@ static void free_repo_appdata(struct pakfire_repo_appdata* appdata) {
        if (appdata->mirrorlist_url)
                free(appdata->mirrorlist_url);
 
+       if (appdata->mirrors)
+               pakfire_mirrorlist_unref(appdata->mirrors);
+
        free(appdata);
 }
 
@@ -351,6 +356,11 @@ PAKFIRE_EXPORT PakfireRepo pakfire_repo_create(Pakfire pakfire, const char* name
                goto ERROR;
        }
 
+       // Allocate mirrorlist
+       r = pakfire_mirrorlist_create(&repo->appdata->mirrors, repo->pakfire);
+       if (r)
+               goto ERROR;
+
        // Setup/clear repository data
        r = pakfire_repo_clear(repo);
        if (r)
@@ -843,7 +853,7 @@ int pakfire_repo_download_packages(PakfireRepo repo, PakfirePackageList packages
 
                // Add transfer to downloader
                r = pakfire_downloader_add_transfer(downloader,
-                       pakfire_package_get_filename(pkg), cache_path);
+                       repo->appdata->mirrors, pakfire_package_get_filename(pkg), cache_path);
                if (r) {
                        pakfire_package_unref(pkg);
                        free(cache_path);

diff --git a/tests/libpakfire/downloader.c b/tests/libpakfire/downloader.c
index 53f19efd432074239d22d41409073e335075891c..a5353dcd8f8f65e476896e7a748a69e99e57ae64 100644 (file)
--- a/tests/libpakfire/downloader.c
+++ b/tests/libpakfire/downloader.c
@@ -36,7 +36,7 @@ static int test_simple(const struct test* t) {
        ASSERT(r == 0);
 
        // Retrieve a file
-       r = pakfire_downloader_retrieve(d, DOWNLOAD_URL, DOWNLOAD_PATH);
+       r = pakfire_downloader_retrieve(d, NULL, DOWNLOAD_URL, DOWNLOAD_PATH);
        ASSERT(r == 0);
 
        // Cleanup
@@ -53,11 +53,11 @@ static int test_retrieve_with_pending_transfers(const struct test* t) {
        ASSERT(r == 0);
 
        // Add a transfer
-       r = pakfire_downloader_add_transfer(d, DOWNLOAD_URL, DOWNLOAD_PATH);
+       r = pakfire_downloader_add_transfer(d, NULL, DOWNLOAD_URL, DOWNLOAD_PATH);
        ASSERT(r == 0);
 
        // Retrieve a file
-       r = pakfire_downloader_retrieve(d, DOWNLOAD_URL, DOWNLOAD_PATH);
+       r = pakfire_downloader_retrieve(d, NULL, DOWNLOAD_URL, DOWNLOAD_PATH);
        ASSERT(r == EBUSY);
 
        // Cleanup