From: Michael Tremer Date: Thu, 9 Mar 2023 20:34:58 +0000 (+0000) Subject: repositories: Refactor how we are reading metadata X-Git-Tag: 0.9.29~342 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=69e81a094b90ba5fbc359fc0625709d24c730d94;p=people%2Fms%2Fpakfire.git repositories: Refactor how we are reading metadata This is a large rewrite of how we are discovering and reading any repository metadata. It first of all makes the code a little bit more straight forward by breaking steps into their own function. Those functions will now do "the right thing" depending whether we are dealing with a local or remote repository and will try to read repository metdata for local repositories, too. If that fails, we will of course fall back and scan. Signed-off-by: Michael Tremer --- diff --git a/src/libpakfire/repo.c b/src/libpakfire/repo.c index 74ea6f570..a61e207dd 100644 --- a/src/libpakfire/repo.c +++ b/src/libpakfire/repo.c @@ -58,8 +58,6 @@ struct pakfire_repo_appdata { char* description; char* baseurl; - char metadata[PATH_MAX]; - // Key fingerprint char key[PAKFIRE_KEY_FPR_MAXLEN]; @@ -108,6 +106,33 @@ static int pakfire_repo_is_commandline(struct pakfire_repo* repo) { return pakfire_repo_name_equals(repo, PAKFIRE_REPO_COMMANDLINE); } +#define pakfire_repo_path(repo, path, format, ...) \ + __pakfire_repo_path(repo, path, sizeof(path), format, __VA_ARGS__) + +static int __pakfire_repo_path(struct pakfire_repo* repo, + char* path, const size_t length, const char* format, ...) { + char buffer[PATH_MAX]; + va_list args; + int r; + + // Format the path + va_start(args, format); + r = __pakfire_string_vformat(buffer, sizeof(buffer), format, args); + va_end(args); + if (r) + return r; + + // For local repositories return the local path + if (pakfire_repo_is_local(repo)) + return __pakfire_string_format(path, length, + "%s/%s", pakfire_repo_get_path(repo), buffer); + + const char* name = pakfire_repo_get_name(repo); + + // Otherwise return the cached path + return __pakfire_cache_path(repo->pakfire, path, length, "%s/%s", name, buffer); +} + static int pakfire_repo_retrieve( struct pakfire_repo* repo, const char* title, @@ -306,99 +331,118 @@ struct pakfire_mirrorlist* pakfire_repo_get_mirrorlist(struct pakfire_repo* repo return pakfire_mirrorlist_ref(repo->mirrorlist); } -static int pakfire_repo_download_database(struct pakfire_repo* repo, const char* database, - const char* cache_path) { +static int pakfire_repo_download_database(struct pakfire_repo* repo, + const char* filename, const char* path) { char title[NAME_MAX]; - char database_url[PATH_MAX]; + char url[PATH_MAX]; + int r; // Do nothing if the file already exists - if (pakfire_path_exists(cache_path)) { - DEBUG(repo->pakfire, "Database %s already present. Skipping download\n", database); + if (pakfire_path_exists(path)) { + DEBUG(repo->pakfire, "Database %s already present. Skipping download\n", filename); return 0; } const char* name = pakfire_repo_get_name(repo); // Make title - pakfire_string_format(title, _("Package Database: %s"), name); + r = pakfire_string_format(title, _("Package Database: %s"), name); + if (r) + return r; // Make download URL - pakfire_string_format(database_url, "repodata/%s", database); + r = pakfire_string_format(url, "repodata/%s", filename); + if (r) + return r; - return pakfire_repo_retrieve(repo, title, database_url, cache_path, - 0, NULL, 0, 0); + return pakfire_repo_retrieve(repo, title, url, path, 0, NULL, 0, 0); } -static int pakfire_repo_read_metadata(struct pakfire_repo* repo, const char* path, int refresh) { +static int pakfire_repo_read_database(struct pakfire_repo* repo, const char* path) { FILE* f = NULL; int r; + DEBUG(repo->pakfire, "Read package database from %s...\n", path); + + // Open the database file + f = fopen(path, "r"); + if (!f) { + ERROR(repo->pakfire, "Could not open package database at %s: %m\n", path); + r = 1; + goto ERROR; + } + + // Drop any previous data + r = pakfire_repo_clear(repo); + if (r) + goto ERROR; + + // Read database + r = pakfire_repo_read_solv(repo, f, 0); + if (r) + goto ERROR; + +ERROR: + if (f) + fclose(f); + + return r; +} + +static int pakfire_repo_read_metadata(struct pakfire_repo* repo, const char* path) { + char database_path[PATH_MAX]; + int r; + + DEBUG(repo->pakfire, "Reading repository metadata from %s...\n", path); + struct json_object* json = pakfire_json_parse_from_file(repo->pakfire, path); if (!json) { - // Ignore metadata not existing when not on refresh mode - if (!refresh && errno == ENOENT) - return 0; + switch (errno) { + case ENOENT: + // If the repository is local, we will just scan it + if (pakfire_repo_is_local(repo)) { + DEBUG(repo->pakfire, "No metadata available on local repository." + " Falling back to scan...\n"); + + return pakfire_repo_scan(repo, 0); + } + break; + } ERROR(repo->pakfire, "Could not parse metadata from %s: %m\n", path); return 1; } - char database_filename[NAME_MAX] = ""; - char database_cache_path[PATH_MAX]; - struct json_object* database = NULL; // Search for the database name int found = json_object_object_get_ex(json, "database", &database); - if (found) { - pakfire_string_set(database_filename, json_object_get_string(database)); - - DEBUG(repo->pakfire, "Using package database %s\n", database_filename); + if (!found) { + ERROR(repo->pakfire, "Could not read database name from metadata\n"); + r = 1; + goto ERROR; } - // Try loading the database - if (*database_filename) { - r = pakfire_cache_path(repo->pakfire, database_cache_path, - "repodata/%s/%s", pakfire_repo_get_name(repo), database_filename); - if (r) - goto ERROR; + const char* filename = json_object_get_string(database); - // Download the database if necessary - if (refresh) { - r = pakfire_repo_download_database(repo, database_filename, database_cache_path); - if (r) - goto ERROR; - } + // Make the path to the database + r = pakfire_repo_path(repo, database_path, "repodata/%s", filename); + if (r) + goto ERROR; - // Drop any previous data - r = pakfire_repo_clear(repo); + // Download the database + if (!pakfire_repo_is_local(repo) && !pakfire_has_flag(repo->pakfire, PAKFIRE_FLAGS_OFFLINE)) { + r = pakfire_repo_download_database(repo, filename, database_path); if (r) goto ERROR; - - // Open database file - f = fopen(database_cache_path, "r"); - if (!f) - goto ERROR; - - // Read database - r = pakfire_repo_read_solv(repo, f, 0); - if (r) { - if (!refresh && errno == ENOENT) - goto NOERROR; - - ERROR(repo->pakfire, "Could not read SOLV file %s: %m\n", database_cache_path); - goto ERROR; - } } -NOERROR: - // Success - r = 0; + // Read the database + r = pakfire_repo_read_database(repo, database_path); + if (r) + goto ERROR; ERROR: - if (f) - fclose(f); - // Free the parsed JSON object json_object_put(json); @@ -406,10 +450,26 @@ ERROR: } static int pakfire_repo_refresh_mirrorlist(struct pakfire_repo* repo, const int force) { + int r; + + // Local repositories don't need a mirrorlist + if (pakfire_repo_is_local(repo)) + return 0; + + // Do nothing if running in offline mode + if (pakfire_has_flag(repo->pakfire, PAKFIRE_FLAGS_OFFLINE)) + return 0; + // This repository does not have a mirrorlist - if (!repo->appdata->mirrorlist_url || !*repo->appdata->mirrorlist) + if (!repo->appdata->mirrorlist_url) return 0; + // Make path to mirrorlist + r = pakfire_cache_path(repo->pakfire, repo->appdata->mirrorlist, + "repodata/%s/mirrorlist", pakfire_repo_get_name(repo)); + if (r) + return r; + // Check if this needs to be refreshed if (!force) { time_t age = pakfire_path_age(repo->appdata->mirrorlist); @@ -425,10 +485,10 @@ static int pakfire_repo_refresh_mirrorlist(struct pakfire_repo* repo, const int 0, NULL, 0, PAKFIRE_TRANSFER_NOPROGRESS); } -static int pakfire_repo_refresh_metadata(struct pakfire_repo* repo, const int force) { +static int pakfire_repo_download_metadata(struct pakfire_repo* repo, const char* path, const int force) { // Check if this needs to be refreshed if (!force) { - time_t age = pakfire_path_age(repo->appdata->metadata); + time_t age = pakfire_path_age(path); if (age > 0 && age < REFRESH_AGE_METADATA) { DEBUG(repo->pakfire, "Skip refreshing metadata which is %lds old\n", age); @@ -436,20 +496,28 @@ static int pakfire_repo_refresh_metadata(struct pakfire_repo* repo, const int fo } } - int r = pakfire_repo_retrieve(repo, NULL, - "repodata/repomd.json", repo->appdata->metadata, 0, - NULL, 0, PAKFIRE_TRANSFER_NOPROGRESS); + return pakfire_repo_retrieve(repo, NULL, "repodata/repomd.json", path, + 0, NULL, 0, PAKFIRE_TRANSFER_NOPROGRESS); +} + +static int pakfire_repo_refresh_metadata(struct pakfire_repo* repo, const int force) { + char path[PATH_MAX]; + int r; + + // Make the metadata path + r = pakfire_repo_path(repo, path, "%s", "repodata/repomd.json"); if (r) return r; - // Parse metadata - r = pakfire_repo_read_metadata(repo, repo->appdata->metadata, 1); - if (r) { - unlink(repo->appdata->metadata); - return r; + // Don't try to refresh the metadata for local repositories and if we are running offline + if (!pakfire_repo_is_local(repo) && !pakfire_has_flag(repo->pakfire, PAKFIRE_FLAGS_OFFLINE)) { + r = pakfire_repo_download_metadata(repo, path, force); + if (r) + return r; } - return 0; + // Parse metadata + return pakfire_repo_read_metadata(repo, path); } static void free_repo_appdata(struct pakfire_repo_appdata* appdata) { @@ -532,29 +600,6 @@ PAKFIRE_EXPORT int pakfire_repo_create(struct pakfire_repo** repo, if (r) goto ERROR; - // Skip remaining initialization for "internal" repositories - if (pakfire_repo_is_internal(rep)) - goto SUCCESS; - - // Make path to mirrorlist - r = pakfire_cache_path(pakfire, rep->appdata->mirrorlist, - "repodata/%s/mirrorlist", pakfire_repo_get_name(rep)); - if (r) - goto ERROR; - - // Make path for metadata - r = pakfire_cache_path(pakfire, rep->appdata->metadata, - "repodata/%s/repomd.json", pakfire_repo_get_name(rep)); - if (r) - goto ERROR; - - // Try loading metadata - r = pakfire_repo_read_metadata(rep, rep->appdata->metadata, 0); - if (r) { - ERROR(rep->pakfire, "Could not initialize repository metadata: %m\n"); - goto ERROR; - } - SUCCESS: *repo = rep; return 0; @@ -1306,14 +1351,6 @@ PAKFIRE_EXPORT int pakfire_repo_refresh(struct pakfire_repo* repo, int force) { return 0; } - // Scan local repositories - if (pakfire_repo_is_local(repo)) - return pakfire_repo_scan(repo, 0); - - // Do nothing if running in offline mode - if (pakfire_has_flag(repo->pakfire, PAKFIRE_FLAGS_OFFLINE)) - return 0; - // Refresh mirrorlist r = pakfire_repo_refresh_mirrorlist(repo, force); if (r) {