From: Michael Tremer Date: Fri, 12 Mar 2021 09:54:20 +0000 (+0000) Subject: repo: Implement downloading mirrorlists X-Git-Tag: 0.9.28~1285^2~558 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=cd2f27a075155eeb2ae08428e55db38fc9a8e614;p=pakfire.git repo: Implement downloading mirrorlists Signed-off-by: Michael Tremer --- diff --git a/src/_pakfire/repo.c b/src/_pakfire/repo.c index 169fd19c0..106091b78 100644 --- a/src/_pakfire/repo.c +++ b/src/_pakfire/repo.c @@ -216,6 +216,23 @@ static int Repo_set_mirrorlist(RepoObject* self, PyObject* value) { return pakfire_repo_set_mirrorlist(self->repo, mirrorlist); } +static PyObject* Repo_refresh(RepoObject* self, PyObject* args, PyObject* kwds) { + char* kwlist[] = { "force", NULL }; + + int force = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|p", kwlist, &force)) + return NULL; + + int r = pakfire_repo_refresh(self->repo, force); + if (r) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + Py_RETURN_NONE; +} + static PyObject* Repo_get_config(RepoObject* self) { char* config = pakfire_repo_get_config(self->repo); @@ -478,6 +495,12 @@ static struct PyMethodDef Repo_methods[] = { METH_VARARGS, NULL }, + { + "refresh", + (PyCFunction)Repo_refresh, + METH_VARARGS|METH_KEYWORDS, + NULL, + }, { "write_solv", (PyCFunction)Repo_write_solv, diff --git a/src/libpakfire/downloader.c b/src/libpakfire/downloader.c index 426020416..b9ed4b41d 100644 --- a/src/libpakfire/downloader.c +++ b/src/libpakfire/downloader.c @@ -210,6 +210,19 @@ static void pakfire_downloader_mirrors_sort(struct pakfire_downloader* downloade pakfire_downloader_mirrors_cmp); } +int pakfire_downloader_read_mirrorlist(struct pakfire_downloader* downloader, + const char* path) { + DEBUG(downloader->pakfire, "Reading mirrorlist from %s\n", path); + + // Parse JSON from path + + // Clear all existing mirrors + + // Add the new mirrors + + return 0; +} + int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader, const char* url, unsigned int priority) { // Make space for another mirror diff --git a/src/libpakfire/include/pakfire/downloader.h b/src/libpakfire/include/pakfire/downloader.h index 0a8dfddbb..e03f86936 100644 --- a/src/libpakfire/include/pakfire/downloader.h +++ b/src/libpakfire/include/pakfire/downloader.h @@ -35,6 +35,8 @@ struct pakfire_downloader* pakfire_downloader_unref(struct pakfire_downloader* d const char* pakfire_downloader_get_baseurl(struct pakfire_downloader* downloader); void pakfire_downloader_set_baseurl(struct pakfire_downloader* downloader, const char* baseurl); +int pakfire_downloader_read_mirrorlist(struct pakfire_downloader* downloader, + const char* path); int pakfire_downloader_add_mirror(struct pakfire_downloader* downloader, const char* url, unsigned int priority); diff --git a/src/libpakfire/include/pakfire/repo.h b/src/libpakfire/include/pakfire/repo.h index 5e28f5077..48f402649 100644 --- a/src/libpakfire/include/pakfire/repo.h +++ b/src/libpakfire/include/pakfire/repo.h @@ -86,7 +86,7 @@ int pakfire_repo_scan(PakfireRepo repo, int flags); // Refresh -int pakfire_repo_refresh(PakfireRepo repo, int force); +int pakfire_repo_refresh(PakfireRepo repo, const int force); #ifdef PAKFIRE_PRIVATE diff --git a/src/libpakfire/repo.c b/src/libpakfire/repo.c index 933a75511..ce1817734 100644 --- a/src/libpakfire/repo.c +++ b/src/libpakfire/repo.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include #include +#include #include #include #include @@ -52,7 +54,10 @@ struct pakfire_repo_appdata { char* description; char* baseurl; char* keyfile; - char* mirrorlist; + + // Mirrorlist + char* mirrorlist_url; + char mirrorlist[PATH_MAX]; }; struct _PakfireRepo { @@ -60,6 +65,8 @@ struct _PakfireRepo { Repo* repo; struct pakfire_repo_appdata* appdata; int nrefs; + + struct pakfire_downloader* downloader; }; Id pakfire_repo_add_solvable(PakfireRepo repo) { @@ -78,8 +85,8 @@ static void free_repo_appdata(struct pakfire_repo_appdata* appdata) { if (appdata->keyfile) free(appdata->keyfile); - if (appdata->mirrorlist) - free(appdata->mirrorlist); + if (appdata->mirrorlist_url) + free(appdata->mirrorlist_url); free(appdata); } @@ -116,6 +123,11 @@ PAKFIRE_EXPORT PakfireRepo pakfire_repo_create(Pakfire pakfire, const char* name repo->appdata->repodata = repo_add_repodata(repo->repo, REPO_EXTEND_SOLVABLES|REPO_LOCALPOOL|REPO_NO_INTERNALIZE|REPO_NO_LOCATION); + + // Make path to mirrorlist + snprintf(repo->appdata->mirrorlist, sizeof(repo->appdata->mirrorlist) - 1, + "%s/repodata/%s/mirrorlist", pakfire_get_cache_path(repo->pakfire), + pakfire_repo_get_name(repo)); } return repo; @@ -145,6 +157,9 @@ PAKFIRE_EXPORT PakfireRepo pakfire_repo_ref(PakfireRepo repo) { static void pakfire_repo_free(PakfireRepo repo) { DEBUG(repo->pakfire, "Releasing Repo at %p\n", repo); + + if (repo->downloader) + pakfire_downloader_unref(repo->downloader); pakfire_unref(repo->pakfire); free(repo); @@ -328,17 +343,17 @@ PAKFIRE_EXPORT int pakfire_repo_set_keyfile(PakfireRepo repo, const char* keyfil } PAKFIRE_EXPORT const char* pakfire_repo_get_mirrorlist(PakfireRepo repo) { - return repo->appdata->mirrorlist; + return repo->appdata->mirrorlist_url; } PAKFIRE_EXPORT int pakfire_repo_set_mirrorlist(PakfireRepo repo, const char* mirrorlist) { - if (repo->appdata->mirrorlist) - free(repo->appdata->mirrorlist); + if (repo->appdata->mirrorlist_url) + free(repo->appdata->mirrorlist_url); if (mirrorlist) - repo->appdata->mirrorlist = strdup(mirrorlist); + repo->appdata->mirrorlist_url = strdup(mirrorlist); else - repo->appdata->mirrorlist = NULL; + repo->appdata->mirrorlist_url = NULL; return 0; } @@ -699,11 +714,77 @@ ERROR: return r; } -static int pakfire_repo_refresh_mirrorlist(PakfireRepo repo, int force) { - return 0; +static struct pakfire_downloader* pakfire_repo_downloader(PakfireRepo repo) { + const char* mirrorlist = repo->appdata->mirrorlist; + + if (!repo->downloader) { + int r = pakfire_downloader_create(&repo->downloader, repo->pakfire); + if (r) + return NULL; + + // Load all mirrors + if (*mirrorlist) { + r = pakfire_downloader_read_mirrorlist(repo->downloader, mirrorlist); + if (r) + return NULL; + } + } + + return pakfire_downloader_ref(repo->downloader); +} + +static int pakfire_repo_refresh_mirrorlist(PakfireRepo repo, const int force) { + const char* mirrorlist_url = repo->appdata->mirrorlist_url; + const char* mirrorlist = repo->appdata->mirrorlist; + int r; + char path[PATH_MAX]; + + // This repository does not have a mirrorlist + if (!mirrorlist_url || !*mirrorlist) + return 0; + + // Get the downloader + struct pakfire_downloader* downloader = pakfire_repo_downloader(repo); + if (!downloader) + return 1; + + // Make download path + snprintf(path, sizeof(path) - 1, "%s/repodata/%s/.mirrorlist", + pakfire_get_cache_path(repo->pakfire), pakfire_repo_get_name(repo)); + + // Try to retrieve the mirrorlist + r = pakfire_downloader_retrieve(downloader, mirrorlist_url, path); + if (r) + goto ERROR; + + // Parse it + r = pakfire_downloader_read_mirrorlist(downloader, path); + if (r) { + unlink(path); + goto ERROR; + } + + // Drop previous version of the mirrorlist + unlink(mirrorlist); + + // Link the temporary file to the permanent location + r = link(path, mirrorlist); + if (r) { + ERROR(repo->pakfire, "Could not write mirrorlist %s: %s\n", + mirrorlist, strerror(errno)); + goto ERROR; + } + + // Success + r = 0; + +ERROR: + pakfire_downloader_unref(downloader); + + return r; } -static int pakfire_repo_refresh_metadata(PakfireRepo repo, int force) { +static int pakfire_repo_refresh_metadata(PakfireRepo repo, const int force) { return 0; } @@ -711,9 +792,17 @@ static int pakfire_repo_refresh_database(PakfireRepo repo) { return 0; } -PAKFIRE_EXPORT int pakfire_repo_refresh(PakfireRepo repo, int force) { +PAKFIRE_EXPORT int pakfire_repo_refresh(PakfireRepo repo, const int force) { int r; + // Do nothing if this repository is not enabled + int enabled = pakfire_repo_get_enabled(repo); + if (!enabled) { + DEBUG(repo->pakfire, "Skip refreshing repository '%s'\n", + pakfire_repo_get_name(repo)); + return 0; + } + // Refresh mirrorlist r = pakfire_repo_refresh_mirrorlist(repo, force); if (r) diff --git a/src/pakfire/base.py b/src/pakfire/base.py index f9c495d57..1852f3f2a 100644 --- a/src/pakfire/base.py +++ b/src/pakfire/base.py @@ -125,14 +125,7 @@ class Pakfire(_pakfire.Pakfire): def refresh_repositories(self, force=False): for repo in self.repos: - if not repo.enabled: - continue - - if repo == self.installed_repo: - continue - - d = downloaders.RepositoryDownloader(self, repo) - d.refresh(force=force) + repo.refresh(force=force) def check_root_user(self): if not os.getuid() == 0 or not os.getgid() == 0: