From: Michael Tremer Date: Thu, 15 Apr 2021 18:06:06 +0000 (+0000) Subject: dist: Refactor downloading sources X-Git-Tag: 0.9.28~1285^2~357 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=e10e1ad14e8e2b914bcf13dcb13dd1553c61b89e;p=pakfire.git dist: Refactor downloading sources Signed-off-by: Michael Tremer --- diff --git a/src/libpakfire/dist.c b/src/libpakfire/dist.c index 1c6be00d6..a7c162fe6 100644 --- a/src/libpakfire/dist.c +++ b/src/libpakfire/dist.c @@ -108,90 +108,100 @@ ERROR: return r; } -static int pakfire_dist_download_source(Pakfire pakfire, const char* filename, - const char* cache_path) { - struct pakfire_downloader* downloader; - - DEBUG(pakfire, "Downloading %s...\n", filename); - - int r = pakfire_downloader_create(&downloader, pakfire); +static int pakfire_dist_create_downloader_and_mirrorlist( + Pakfire pakfire, PakfireParser makefile, + struct pakfire_downloader** downloader, struct pakfire_mirrorlist** mirrorlist) { + // Create the downloader + int r = pakfire_downloader_create(downloader, pakfire); if (r) { ERROR(pakfire, "Could not initialize downloader\n"); return r; } - // XXX Set baseurl - pakfire_downloader_set_baseurl(downloader, "https://source.ipfire.org/source-3.x/"); + // Set base URL (XXX) + pakfire_downloader_set_baseurl(*downloader, "https://source.ipfire.org/source-3.x/"); - // Add download - r = pakfire_downloader_add_transfer(downloader, NULL, filename, cache_path); - if (r) - goto ERROR; + // Fetch source_dl + char** source_dls = pakfire_parser_get_split(makefile, NULL, "source_dl", ' '); - // Perform download - r = pakfire_downloader_run(downloader); - if (r) + // We do not need to create a mirrorlist if this isn't set + if (!source_dls) + return 0; + + // Create mirrorlist + r = pakfire_mirrorlist_create(mirrorlist, pakfire); + if (r) { + ERROR(pakfire, "Could not create the mirrorlist\n"); goto ERROR; + } + + // Add all mirrors + for (char** source_dl = source_dls; *source_dl; source_dl++) { + r = pakfire_mirrorlist_add_mirror(*mirrorlist, *source_dl); + if (r) + goto ERROR; + } // Success r = 0; ERROR: - pakfire_downloader_unref(downloader); + if (source_dls) { + for (char** source_dl = source_dls; *source_dl; source_dl++) + free(*source_dl); + free(source_dls); + } return r; } static int pakfire_dist_add_source(Pakfire pakfire, struct pakfire_packager* packager, - PakfirePackage pkg, const char* filename) { + PakfirePackage pkg, struct pakfire_downloader* downloader, + struct pakfire_mirrorlist* mirrorlist, const char* filename) { int r; char archive_path[PATH_MAX]; char cache_path[PATH_MAX]; const char* name = pakfire_package_get_name(pkg); - - snprintf(archive_path, sizeof(archive_path) - 1, "files/%s", filename); pakfire_make_cache_path(pakfire, cache_path, sizeof(cache_path) - 1, "sources/%s/%s", name, filename); // Download the file if it does not exist in the cache if (access(cache_path, R_OK) != 0) { - r = pakfire_dist_download_source(pakfire, filename, cache_path); + r = pakfire_downloader_retrieve(downloader, mirrorlist, filename, cache_path); if (r) return r; } + pakfire_string_format(archive_path, "files/%s", filename); + // Add file to package return pakfire_packager_add(packager, cache_path, archive_path); } static int pakfire_dist_add_sources(Pakfire pakfire, struct pakfire_packager* packager, PakfirePackage pkg, PakfireParser makefile) { - int r; + // Fetch sources + char** sources = pakfire_parser_get_split(makefile, NULL, "sources", ' '); - char* sources = pakfire_parser_get(makefile, NULL, "sources"); + // Nothing to do if this variable is empty if (!sources) return 1; - // Nothing to do if there are no sources - if (!*sources) { - free(sources); - return 0; - } + struct pakfire_downloader* downloader = NULL; + struct pakfire_mirrorlist* mirrorlist = NULL; - // Split sources by space - char** sourceslist = pakfire_split_string(sources, ' '); - if (!sourceslist) { - ERROR(pakfire, "Could not split sources: %s\n", strerror(errno)); - r = 1; + // Create a downloader and mirrorlist + int r = pakfire_dist_create_downloader_and_mirrorlist(pakfire, makefile, + &downloader, &mirrorlist); + if (r) goto ERROR; - } // Add all files one by one - for (char** source = sourceslist; *source; source++) { + for (char** source = sources; *source; source++) { DEBUG(pakfire, "Adding source file %s\n", *source); - r = pakfire_dist_add_source(pakfire, packager, pkg, *source); + r = pakfire_dist_add_source(pakfire, packager, pkg, downloader, mirrorlist, *source); if (r) { ERROR(pakfire, "Could not add '%s' to package: %s\n", *source, strerror(errno)); goto ERROR; @@ -202,10 +212,15 @@ static int pakfire_dist_add_sources(Pakfire pakfire, struct pakfire_packager* pa r = 0; ERROR: - if (sourceslist) { - for (char** source = sourceslist; *source; source++) + if (downloader) + pakfire_downloader_unref(downloader); + if (mirrorlist) + pakfire_mirrorlist_unref(mirrorlist); + + if (sources) { + for (char** source = sources; *source; source++) free(*source); - free(sourceslist); + free(sources); } return r; diff --git a/src/libpakfire/include/pakfire/parser.h b/src/libpakfire/include/pakfire/parser.h index 9e2946791..b4083f935 100644 --- a/src/libpakfire/include/pakfire/parser.h +++ b/src/libpakfire/include/pakfire/parser.h @@ -47,6 +47,8 @@ int pakfire_parser_append(PakfireParser parser, char* pakfire_parser_expand(PakfireParser parser, const char* namespace, const char* value); char* pakfire_parser_get(PakfireParser parser, const char* namespace, const char* name); +char** pakfire_parser_get_split(PakfireParser parser, + const char* namespace, const char* name, char delim); int pakfire_parser_merge(PakfireParser parser1, PakfireParser parser2); diff --git a/src/libpakfire/libpakfire.sym b/src/libpakfire/libpakfire.sym index 8ec3c234c..5181eac6c 100644 --- a/src/libpakfire/libpakfire.sym +++ b/src/libpakfire/libpakfire.sym @@ -260,6 +260,7 @@ global: pakfire_parser_get; pakfire_parser_get_namespace; pakfire_parser_get_parent; + pakfire_parser_get_split; pakfire_parser_merge; pakfire_parser_parse; pakfire_parser_read; diff --git a/src/libpakfire/parser.c b/src/libpakfire/parser.c index d660a39e4..91c971031 100644 --- a/src/libpakfire/parser.c +++ b/src/libpakfire/parser.c @@ -609,6 +609,19 @@ PAKFIRE_EXPORT char* pakfire_parser_get(PakfireParser parser, const char* namesp return pakfire_parser_expand(parser, namespace, value); } +PAKFIRE_EXPORT char** pakfire_parser_get_split(PakfireParser parser, + const char* namespace, const char* name, char delim) { + char* value = pakfire_parser_get(parser, namespace, name); + if (!value) + return NULL; + + // Split the string + char** list = pakfire_split_string(value, delim); + free(value); + + return list; +} + PAKFIRE_EXPORT int pakfire_parser_merge(PakfireParser parser1, PakfireParser parser2) { DEBUG(parser1->pakfire, "Merging parsers %p and %p\n", parser1, parser2);