From 6c0217d3139dc86085227c7fd936efcf589282e8 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Wed, 16 Aug 2023 15:37:33 +0000 Subject: [PATCH] repo: Fix file structure Because it is possible to have multiple builds with the same nevra in the same repository, we need to store packages in a subdirectory to be able to uniquely identify them. That is also helpful when we want to avoid downloading a package when expecting a different UUID. This patch changes that packages will now be put into some subdirectory structure so multiple archives with the same name can coexist. The downloader and caching mechanism for repositories had to be tweaked to honour the subdirectory structure. Signed-off-by: Michael Tremer --- src/libpakfire/include/pakfire/package.h | 1 + src/libpakfire/package.c | 86 ++++++++++++++++++------ src/libpakfire/repo.c | 45 ++++++++++--- src/libpakfire/transaction.c | 47 ++++++++----- 4 files changed, 133 insertions(+), 46 deletions(-) diff --git a/src/libpakfire/include/pakfire/package.h b/src/libpakfire/include/pakfire/package.h index 64074fdf6..f584eac54 100644 --- a/src/libpakfire/include/pakfire/package.h +++ b/src/libpakfire/include/pakfire/package.h @@ -49,6 +49,7 @@ enum pakfire_package_key { PAKFIRE_PKG_DISTRO, PAKFIRE_PKG_PACKAGER, PAKFIRE_PKG_PATH, + PAKFIRE_PKG_CACHE_PATH, PAKFIRE_PKG_FILENAME, PAKFIRE_PKG_DOWNLOADSIZE, PAKFIRE_PKG_INSTALLSIZE, diff --git a/src/libpakfire/package.c b/src/libpakfire/package.c index 7f4bdfd62..2658a7d6a 100644 --- a/src/libpakfire/package.c +++ b/src/libpakfire/package.c @@ -61,6 +61,7 @@ struct pakfire_package { char filename[NAME_MAX]; char path[PATH_MAX]; + char cache_path[PATH_MAX]; }; static Solvable* get_solvable(struct pakfire_package* pkg) { @@ -415,20 +416,26 @@ static const char* pakfire_package_make_filename(struct pakfire_package* pkg) { } static int pakfire_package_make_cache_path(struct pakfire_package* pkg) { - const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME); + const char* nevra = pakfire_package_get_string(pkg, PAKFIRE_PKG_NEVRA); - enum pakfire_digest_types digest_type = PAKFIRE_DIGEST_UNDEFINED; - size_t digest_length = 0; + // Fetch the filename + const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME); + if (!filename) { + ERROR(pkg->pakfire, "Could not fetch filename for %s: %m\n", nevra); + return 1; + } - // Fetch the digest - const unsigned char* digest = pakfire_package_get_digest(pkg, - &digest_type, &digest_length); + // Fetch UUID + const char* uuid = pakfire_package_get_string(pkg, PAKFIRE_PKG_UUID); + if (!uuid) { + ERROR(pkg->pakfire, "Could not fetch UUID for %s: %m\n", nevra); + return 1; + } - if (digest && digest_length >= 4) - return pakfire_cache_path(pkg->pakfire, pkg->path, - "%02x/%02x/%02x/%02x/%s", digest[0], digest[1], digest[2], digest[3], filename); + // XXX check if the UUID is valid - return pakfire_cache_path(pkg->pakfire, pkg->path, "%s", filename); + return pakfire_cache_path(pkg->pakfire, pkg->cache_path, "%c%c/%c%c/%s/%s", + uuid[0], uuid[1], uuid[2], uuid[4], uuid + 5, filename); } PAKFIRE_EXPORT const char* pakfire_package_get_string( @@ -498,15 +505,18 @@ PAKFIRE_EXPORT const char* pakfire_package_get_string( case PAKFIRE_PKG_PATH: if (!*pkg->path) { - const char* base = solvable_lookup_str(s, SOLVABLE_MEDIABASE); - if (base) { - const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME); - if (!filename) - return NULL; + const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME); + if (!filename) + return NULL; - pakfire_string_format(pkg->path, "%s/%s", base, filename); + // XXX should be MEDIADIR + const char* dir = solvable_lookup_str(s, SOLVABLE_MEDIABASE); + if (dir) { + r = pakfire_string_format(pkg->path, "%s/%s", dir, filename); + if (r) + return NULL; } else { - r = pakfire_package_make_cache_path(pkg); + r = pakfire_string_set(pkg->path, filename); if (r) return NULL; } @@ -522,6 +532,15 @@ PAKFIRE_EXPORT const char* pakfire_package_get_string( ret = pakfire_package_make_filename(pkg); break; + case PAKFIRE_PKG_CACHE_PATH: + if (!*pkg->cache_path) { + r = pakfire_package_make_cache_path(pkg); + if (r) + return NULL; + } + + return pkg->cache_path; + case PAKFIRE_PKG_BUILD_HOST: ret = solvable_lookup_str(s, SOLVABLE_BUILDHOST); break; @@ -655,6 +674,7 @@ PAKFIRE_EXPORT int pakfire_package_set_string( else solvable_unset(s, SOLVABLE_MEDIAFILE); + // XXX this should be MEDIADIR if (dirname) solvable_set_str(s, SOLVABLE_MEDIABASE, dirname); else @@ -672,6 +692,9 @@ PAKFIRE_EXPORT int pakfire_package_set_string( id = SOLVABLE_MEDIAFILE; break; + case PAKFIRE_PKG_CACHE_PATH: + return pakfire_string_set(pkg->cache_path, value); + case PAKFIRE_PKG_BUILD_HOST: id = SOLVABLE_BUILDHOST; break; @@ -1419,16 +1442,35 @@ PAKFIRE_EXPORT char* pakfire_package_dump(struct pakfire_package* pkg, int flags PAKFIRE_EXPORT struct pakfire_archive* pakfire_package_get_archive(struct pakfire_package* pkg) { struct pakfire_archive* archive = NULL; + struct pakfire_repo* repo = NULL; + const char* path = NULL; + int r; + + // Fetch the repository + repo = pakfire_package_get_repo(pkg); + if (repo) { + if (pakfire_repo_is_local(repo)) { + path = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH); + if (!path) + goto ERROR; + } + } // Otherwise open the archive from the cache - const char* path = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH); if (!path) - return NULL; + path = pakfire_package_get_string(pkg, PAKFIRE_PKG_CACHE_PATH); // Open archive - int r = pakfire_archive_open(&archive, pkg->pakfire, path); - if (r) - return NULL; + r = pakfire_archive_open(&archive, pkg->pakfire, path); + if (r) { + ERROR(pkg->pakfire, "Could not open archive for %s (at %s): %m\n", + pakfire_package_get_string(pkg, PAKFIRE_PKG_NEVRA), path); + goto ERROR; + } + +ERROR: + if (repo) + pakfire_repo_unref(repo); return archive; } diff --git a/src/libpakfire/repo.c b/src/libpakfire/repo.c index 4e51cfb99..2fbbc8e6b 100644 --- a/src/libpakfire/repo.c +++ b/src/libpakfire/repo.c @@ -89,7 +89,15 @@ int pakfire_repo_is_internal(struct pakfire_repo* repo) { return (*name == '@'); } +static int pakfire_repo_is_commandline(struct pakfire_repo* repo) { + return pakfire_repo_name_equals(repo, PAKFIRE_REPO_COMMANDLINE); +} + int pakfire_repo_is_local(struct pakfire_repo* repo) { + // The commandline repository is semi-local + if (pakfire_repo_is_commandline(repo)) + return 1; + if (!repo->appdata->baseurl) return 0; @@ -104,10 +112,6 @@ int pakfire_repo_name_equals(struct pakfire_repo* repo, const char* name) { return strcmp(n, name) == 0; } -static int pakfire_repo_is_commandline(struct pakfire_repo* repo) { - return pakfire_repo_name_equals(repo, PAKFIRE_REPO_COMMANDLINE); -} - #define pakfire_repo_path(repo, path, format, ...) \ __pakfire_repo_path(repo, path, sizeof(path), format, __VA_ARGS__) @@ -1661,7 +1665,10 @@ PAKFIRE_EXPORT int pakfire_repo_compose(struct pakfire* pakfire, const char* pat struct pakfire_archive* archive = NULL; struct pakfire_package* package = NULL; + const char* uuid = NULL; + const char* filename = NULL; char destination_path[PATH_MAX]; + char repo_path[PATH_MAX]; DEBUG(pakfire, "Adding %zu file(s) to the repository\n", num_files); @@ -1684,10 +1691,32 @@ PAKFIRE_EXPORT int pakfire_repo_compose(struct pakfire* pakfire, const char* pat goto OUT; } - const char* filename = pakfire_package_get_string(package, PAKFIRE_PKG_FILENAME); + // Fetch the UUID + uuid = pakfire_package_get_string(package, PAKFIRE_PKG_UUID); + if (!uuid) { + ERROR(pakfire, "Could not retrieve the UUID of %s: %m\n", *file); + r = -EINVAL; + goto OUT; + } + + // Fetch the filename + filename = pakfire_package_get_string(package, PAKFIRE_PKG_FILENAME); + if (!filename) { + ERROR(pakfire, "Could not retrieve filename of %s: %m\n", *file); + r = -EINVAL; + goto OUT; + } + + // XXX check if the UUID is valid + + // Make new path where the archive is being stored + r = pakfire_string_format(repo_path, "%c%c/%c%c/%s/%s", + uuid[0], uuid[1], uuid[2], uuid[3], uuid + 4, filename); + if (r) + goto OUT; - // Make new path - r = pakfire_path_join(destination_path, realpath, filename); + // Make the path absolute + r = pakfire_path_join(destination_path, realpath, repo_path); if (r) goto OUT; @@ -1699,7 +1728,7 @@ PAKFIRE_EXPORT int pakfire_repo_compose(struct pakfire* pakfire, const char* pat } // Reset the path - pakfire_package_set_string(package, PAKFIRE_PKG_PATH, NULL); + pakfire_package_set_string(package, PAKFIRE_PKG_PATH, repo_path); OUT: if (archive) { diff --git a/src/libpakfire/transaction.c b/src/libpakfire/transaction.c index 4f2736330..244529a8e 100644 --- a/src/libpakfire/transaction.c +++ b/src/libpakfire/transaction.c @@ -1064,11 +1064,8 @@ static int pakfire_transaction_open_archives(struct pakfire_transaction* transac } transaction->archives[i] = pakfire_package_get_archive(pkg); - if (!transaction->archives[i]) { - ERROR(transaction->pakfire, "Could not open archive for %s: %m\n", - pakfire_package_get_string(pkg, PAKFIRE_PKG_NEVRA)); + if (!transaction->archives[i]) return 1; - } } return 0; @@ -1221,17 +1218,17 @@ static int pakfire_transaction_download_package(struct pakfire_transaction* tran if (!nevra) goto ERROR; - // Where to store the package? + // Where do we find the package on the server? const char* path = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH); if (!path) { ERROR(transaction->pakfire, "Could not retrieve package path for %s: %m\n", nevra); goto ERROR; } - // What file to download? - const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME); - if (!filename) { - ERROR(transaction->pakfire, "Could not retrieve filename for package %s: %m\n", nevra); + // Where do we store the package? + const char* cache_path = pakfire_package_get_string(pkg, PAKFIRE_PKG_CACHE_PATH); + if (!cache_path) { + ERROR(transaction->pakfire, "Could not retrieve package cache path for %s: %m\n", nevra); goto ERROR; } @@ -1247,7 +1244,7 @@ static int pakfire_transaction_download_package(struct pakfire_transaction* tran // Add transfer to downloader r = pakfire_downloader_add_transfer(downloader, baseurl, mirrorlist, - nevra, filename, path, digest_type, digest, digest_length, 0); + nevra, path, cache_path, digest_type, digest, digest_length, 0); ERROR: if (mirrorlist) @@ -1260,6 +1257,9 @@ ERROR: static int pakfire_transaction_package_needs_download( struct pakfire_transaction* transaction, struct pakfire_package* pkg) { + struct pakfire_repo* repo = NULL; + int r = 0; + enum pakfire_steps type = pakfire_transaction_get_step_type(transaction, pkg); switch (type) { case PAKFIRE_STEP_INSTALL: @@ -1270,22 +1270,37 @@ static int pakfire_transaction_package_needs_download( // No need to download for these steps default: - return 0; + goto END; } // No download required if this package is already installed if (pakfire_package_is_installed(pkg)) - return 0; + goto END; - const char* path = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH); + // Fetch the repository + repo = pakfire_package_get_repo(pkg); + if (repo) { + // We don't need to download for local repositories + if (pakfire_repo_is_local(repo)) + goto END; + } + + // Otherwise we check if the archive is already cached + const char* path = pakfire_package_get_string(pkg, PAKFIRE_PKG_CACHE_PATH); // Does the file exist? - int r = access(path, R_OK); + r = access(path, R_OK); if (r == 0) - return 0; + goto END; // This package needs to be downloaded - return 1; + r = 1; + +END: + if (repo) + pakfire_repo_unref(repo); + + return r; } PAKFIRE_EXPORT int pakfire_transaction_download(struct pakfire_transaction* transaction) { -- 2.39.5