repo: Fix file structure

Because it is possible to have multiple builds with the same nevra in
the same repository, we need to store packages in a subdirectory to be
able to uniquely identify them. That is also helpful when we want to
avoid downloading a package when expecting a different UUID.

This patch changes that packages will now be put into some subdirectory
structure so multiple archives with the same name can coexist.

The downloader and caching mechanism for repositories had to be tweaked
to honour the subdirectory structure.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/libpakfire/include/pakfire/package.h b/src/libpakfire/include/pakfire/package.h
index 64074fdf69b175dc1cc18b18dd4cd85c0a761a15..f584eac5400c3f86d15dfe1bf0611f4e5948f526 100644 (file)
--- a/src/libpakfire/include/pakfire/package.h
+++ b/src/libpakfire/include/pakfire/package.h
@@ -49,6 +49,7 @@ enum pakfire_package_key {
        PAKFIRE_PKG_DISTRO,
        PAKFIRE_PKG_PACKAGER,
        PAKFIRE_PKG_PATH,
+       PAKFIRE_PKG_CACHE_PATH,
        PAKFIRE_PKG_FILENAME,
        PAKFIRE_PKG_DOWNLOADSIZE,
        PAKFIRE_PKG_INSTALLSIZE,

diff --git a/src/libpakfire/package.c b/src/libpakfire/package.c
index 7f4bdfd629ba453c7c0102e9221eee0c3d59669f..2658a7d6a95745ef62043d42e5b2fb4dc429a827 100644 (file)
--- a/src/libpakfire/package.c
+++ b/src/libpakfire/package.c
@@ -61,6 +61,7 @@ struct pakfire_package {
 
        char filename[NAME_MAX];
        char path[PATH_MAX];
+       char cache_path[PATH_MAX];
 };
 
 static Solvable* get_solvable(struct pakfire_package* pkg) {
@@ -415,20 +416,26 @@ static const char* pakfire_package_make_filename(struct pakfire_package* pkg) {
 }
 
 static int pakfire_package_make_cache_path(struct pakfire_package* pkg) {
-       const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME);
+       const char* nevra = pakfire_package_get_string(pkg, PAKFIRE_PKG_NEVRA);
 
-       enum pakfire_digest_types digest_type = PAKFIRE_DIGEST_UNDEFINED;
-       size_t digest_length = 0;
+       // Fetch the filename
+       const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME);
+       if (!filename) {
+               ERROR(pkg->pakfire, "Could not fetch filename for %s: %m\n", nevra);
+               return 1;
+       }
 
-       // Fetch the digest
-       const unsigned char* digest = pakfire_package_get_digest(pkg,
-               &digest_type, &digest_length);
+       // Fetch UUID
+       const char* uuid = pakfire_package_get_string(pkg, PAKFIRE_PKG_UUID);
+       if (!uuid) {
+               ERROR(pkg->pakfire, "Could not fetch UUID for %s: %m\n", nevra);
+               return 1;
+       }
 
-       if (digest && digest_length >= 4)
-               return pakfire_cache_path(pkg->pakfire, pkg->path,
-                       "%02x/%02x/%02x/%02x/%s", digest[0], digest[1], digest[2], digest[3], filename);
+       // XXX check if the UUID is valid
 
-       return pakfire_cache_path(pkg->pakfire, pkg->path, "%s", filename);
+       return pakfire_cache_path(pkg->pakfire, pkg->cache_path, "%c%c/%c%c/%s/%s",
+               uuid[0], uuid[1], uuid[2], uuid[4], uuid + 5, filename);
 }
 
 PAKFIRE_EXPORT const char* pakfire_package_get_string(
@@ -498,15 +505,18 @@ PAKFIRE_EXPORT const char* pakfire_package_get_string(
 
                case PAKFIRE_PKG_PATH:
                        if (!*pkg->path) {
-                               const char* base = solvable_lookup_str(s, SOLVABLE_MEDIABASE);
-                               if (base) {
-                                       const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME);
-                                       if (!filename)
-                                               return NULL;
+                               const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME);
+                               if (!filename)
+                                       return NULL;
 
-                                       pakfire_string_format(pkg->path, "%s/%s", base, filename);
+                               // XXX should be MEDIADIR
+                               const char* dir = solvable_lookup_str(s, SOLVABLE_MEDIABASE);
+                               if (dir) {
+                                       r = pakfire_string_format(pkg->path, "%s/%s", dir, filename);
+                                       if (r)
+                                               return NULL;
                                } else {
-                                       r = pakfire_package_make_cache_path(pkg);
+                                       r = pakfire_string_set(pkg->path, filename);
                                        if (r)
                                                return NULL;
                                }
@@ -522,6 +532,15 @@ PAKFIRE_EXPORT const char* pakfire_package_get_string(
                                ret = pakfire_package_make_filename(pkg);
                        break;
 
+               case PAKFIRE_PKG_CACHE_PATH:
+                       if (!*pkg->cache_path) {
+                               r = pakfire_package_make_cache_path(pkg);
+                               if (r)
+                                       return NULL;
+                       }
+
+                       return pkg->cache_path;
+
                case PAKFIRE_PKG_BUILD_HOST:
                        ret = solvable_lookup_str(s, SOLVABLE_BUILDHOST);
                        break;
@@ -655,6 +674,7 @@ PAKFIRE_EXPORT int pakfire_package_set_string(
                        else
                                solvable_unset(s, SOLVABLE_MEDIAFILE);
 
+                       // XXX this should be MEDIADIR
                        if (dirname)
                                solvable_set_str(s, SOLVABLE_MEDIABASE, dirname);
                        else
@@ -672,6 +692,9 @@ PAKFIRE_EXPORT int pakfire_package_set_string(
                        id = SOLVABLE_MEDIAFILE;
                        break;
 
+               case PAKFIRE_PKG_CACHE_PATH:
+                       return pakfire_string_set(pkg->cache_path, value);
+
                case PAKFIRE_PKG_BUILD_HOST:
                        id = SOLVABLE_BUILDHOST;
                        break;
@@ -1419,16 +1442,35 @@ PAKFIRE_EXPORT char* pakfire_package_dump(struct pakfire_package* pkg, int flags
 
 PAKFIRE_EXPORT struct pakfire_archive* pakfire_package_get_archive(struct pakfire_package* pkg) {
        struct pakfire_archive* archive = NULL;
+       struct pakfire_repo* repo = NULL;
+       const char* path = NULL;
+       int r;
+
+       // Fetch the repository
+       repo = pakfire_package_get_repo(pkg);
+       if (repo) {
+               if (pakfire_repo_is_local(repo)) {
+                       path = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH);
+                       if (!path)
+                               goto ERROR;
+               }
+       }
 
        // Otherwise open the archive from the cache
-       const char* path = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH);
        if (!path)
-               return NULL;
+               path = pakfire_package_get_string(pkg, PAKFIRE_PKG_CACHE_PATH);
 
        // Open archive
-       int r = pakfire_archive_open(&archive, pkg->pakfire, path);
-       if (r)
-               return NULL;
+       r = pakfire_archive_open(&archive, pkg->pakfire, path);
+       if (r) {
+               ERROR(pkg->pakfire, "Could not open archive for %s (at %s): %m\n",
+                       pakfire_package_get_string(pkg, PAKFIRE_PKG_NEVRA), path);
+               goto ERROR;
+       }
+
+ERROR:
+       if (repo)
+               pakfire_repo_unref(repo);
 
        return archive;
 }

diff --git a/src/libpakfire/repo.c b/src/libpakfire/repo.c
index 4e51cfb99b16f49da19f31fb9ae77eb4da88c020..2fbbc8e6bc1e10a9c08e29d42bfad9e01f50ec5f 100644 (file)
--- a/src/libpakfire/repo.c
+++ b/src/libpakfire/repo.c
@@ -89,7 +89,15 @@ int pakfire_repo_is_internal(struct pakfire_repo* repo) {
        return (*name == '@');
 }
 
+static int pakfire_repo_is_commandline(struct pakfire_repo* repo) {
+       return pakfire_repo_name_equals(repo, PAKFIRE_REPO_COMMANDLINE);
+}
+
 int pakfire_repo_is_local(struct pakfire_repo* repo) {
+       // The commandline repository is semi-local
+       if (pakfire_repo_is_commandline(repo))
+               return 1;
+
        if (!repo->appdata->baseurl)
                return 0;
 
@@ -104,10 +112,6 @@ int pakfire_repo_name_equals(struct pakfire_repo* repo, const char* name) {
        return strcmp(n, name) == 0;
 }
 
-static int pakfire_repo_is_commandline(struct pakfire_repo* repo) {
-       return pakfire_repo_name_equals(repo, PAKFIRE_REPO_COMMANDLINE);
-}
-
 #define pakfire_repo_path(repo, path, format, ...) \
        __pakfire_repo_path(repo, path, sizeof(path), format, __VA_ARGS__)
 
@@ -1661,7 +1665,10 @@ PAKFIRE_EXPORT int pakfire_repo_compose(struct pakfire* pakfire, const char* pat
 
        struct pakfire_archive* archive = NULL;
        struct pakfire_package* package = NULL;
+       const char* uuid = NULL;
+       const char* filename = NULL;
        char destination_path[PATH_MAX];
+       char repo_path[PATH_MAX];
 
        DEBUG(pakfire, "Adding %zu file(s) to the repository\n", num_files);
 
@@ -1684,10 +1691,32 @@ PAKFIRE_EXPORT int pakfire_repo_compose(struct pakfire* pakfire, const char* pat
                                goto OUT;
                        }
 
-                       const char* filename = pakfire_package_get_string(package, PAKFIRE_PKG_FILENAME);
+                       // Fetch the UUID
+                       uuid = pakfire_package_get_string(package, PAKFIRE_PKG_UUID);
+                       if (!uuid) {
+                               ERROR(pakfire, "Could not retrieve the UUID of %s: %m\n", *file);
+                               r = -EINVAL;
+                               goto OUT;
+                       }
+
+                       // Fetch the filename
+                       filename = pakfire_package_get_string(package, PAKFIRE_PKG_FILENAME);
+                       if (!filename) {
+                               ERROR(pakfire, "Could not retrieve filename of %s: %m\n", *file);
+                               r = -EINVAL;
+                               goto OUT;
+                       }
+
+                       // XXX check if the UUID is valid
+
+                       // Make new path where the archive is being stored
+                       r = pakfire_string_format(repo_path, "%c%c/%c%c/%s/%s",
+                               uuid[0], uuid[1], uuid[2], uuid[3], uuid + 4, filename);
+                       if (r)
+                               goto OUT;
 
-                       // Make new path
-                       r = pakfire_path_join(destination_path, realpath, filename);
+                       // Make the path absolute
+                       r = pakfire_path_join(destination_path, realpath, repo_path);
                        if (r)
                                goto OUT;
 
@@ -1699,7 +1728,7 @@ PAKFIRE_EXPORT int pakfire_repo_compose(struct pakfire* pakfire, const char* pat
                        }
 
                        // Reset the path
-                       pakfire_package_set_string(package, PAKFIRE_PKG_PATH, NULL);
+                       pakfire_package_set_string(package, PAKFIRE_PKG_PATH, repo_path);
 
 OUT:
                        if (archive) {

diff --git a/src/libpakfire/transaction.c b/src/libpakfire/transaction.c
index 4f27363303a8797bca94eec2cf07e56647d8799e..244529a8e41966e171bf404957d484f7c0d1a9ce 100644 (file)
--- a/src/libpakfire/transaction.c
+++ b/src/libpakfire/transaction.c
@@ -1064,11 +1064,8 @@ static int pakfire_transaction_open_archives(struct pakfire_transaction* transac
                }
 
                transaction->archives[i] = pakfire_package_get_archive(pkg);
-               if (!transaction->archives[i]) {
-                       ERROR(transaction->pakfire, "Could not open archive for %s: %m\n",
-                               pakfire_package_get_string(pkg, PAKFIRE_PKG_NEVRA));
+               if (!transaction->archives[i])
                        return 1;
-               }
        }
 
        return 0;
@@ -1221,17 +1218,17 @@ static int pakfire_transaction_download_package(struct pakfire_transaction* tran
        if (!nevra)
                goto ERROR;
 
-       // Where to store the package?
+       // Where do we find the package on the server?
        const char* path = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH);
        if (!path) {
                ERROR(transaction->pakfire, "Could not retrieve package path for %s: %m\n", nevra);
                goto ERROR;
        }
 
-       // What file to download?
-       const char* filename = pakfire_package_get_string(pkg, PAKFIRE_PKG_FILENAME);
-       if (!filename) {
-               ERROR(transaction->pakfire, "Could not retrieve filename for package %s: %m\n", nevra);
+       // Where do we store the package?
+       const char* cache_path = pakfire_package_get_string(pkg, PAKFIRE_PKG_CACHE_PATH);
+       if (!cache_path) {
+               ERROR(transaction->pakfire, "Could not retrieve package cache path for %s: %m\n", nevra);
                goto ERROR;
        }
 
@@ -1247,7 +1244,7 @@ static int pakfire_transaction_download_package(struct pakfire_transaction* tran
 
        // Add transfer to downloader
        r = pakfire_downloader_add_transfer(downloader, baseurl, mirrorlist,
-               nevra, filename, path, digest_type, digest, digest_length, 0);
+               nevra, path, cache_path, digest_type, digest, digest_length, 0);
 
 ERROR:
        if (mirrorlist)
@@ -1260,6 +1257,9 @@ ERROR:
 
 static int pakfire_transaction_package_needs_download(
                struct pakfire_transaction* transaction, struct pakfire_package* pkg) {
+       struct pakfire_repo* repo = NULL;
+       int r = 0;
+
        enum pakfire_steps type = pakfire_transaction_get_step_type(transaction, pkg);
        switch (type) {
                case PAKFIRE_STEP_INSTALL:
@@ -1270,22 +1270,37 @@ static int pakfire_transaction_package_needs_download(
 
                // No need to download for these steps
                default:
-                       return 0;
+                       goto END;
        }
 
        // No download required if this package is already installed
        if (pakfire_package_is_installed(pkg))
-               return 0;
+               goto END;
 
-       const char* path = pakfire_package_get_string(pkg, PAKFIRE_PKG_PATH);
+       // Fetch the repository
+       repo = pakfire_package_get_repo(pkg);
+       if (repo) {
+               // We don't need to download for local repositories
+               if (pakfire_repo_is_local(repo))
+                       goto END;
+       }
+
+       // Otherwise we check if the archive is already cached
+       const char* path = pakfire_package_get_string(pkg, PAKFIRE_PKG_CACHE_PATH);
 
        // Does the file exist?
-       int r = access(path, R_OK);
+       r = access(path, R_OK);
        if (r == 0)
-               return 0;
+               goto END;
 
        // This package needs to be downloaded
-       return 1;
+       r = 1;
+
+END:
+       if (repo)
+               pakfire_repo_unref(repo);
+
+       return r;
 }
 
 PAKFIRE_EXPORT int pakfire_transaction_download(struct pakfire_transaction* transaction) {