From e4fc9122f723b4b08fc39908520812ffd4946965 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Mon, 3 Feb 2025 14:28:16 +0000 Subject: [PATCH] repo: Move sync decision There are actually not a lot of changes, I just needed to move a lot of static functions. Signed-off-by: Michael Tremer --- src/pakfire/repo.c | 594 ++++++++++++++++++++++----------------------- 1 file changed, 293 insertions(+), 301 deletions(-) diff --git a/src/pakfire/repo.c b/src/pakfire/repo.c index f763fe81..a0f4a18f 100644 --- a/src/pakfire/repo.c +++ b/src/pakfire/repo.c @@ -439,6 +439,53 @@ int pakfire_repo_import(struct pakfire* pakfire, struct pakfire_config* config) return 0; } +static int pakfire_repo_to_packagelist( + struct pakfire_repo* self, struct pakfire_packagelist** list) { + struct pakfire_packagelist* l = NULL; + struct pakfire_package* pkg = NULL; + Solvable* s = NULL; + Id id; + int i; + int r; + + // Create a new packagelist + r = pakfire_packagelist_create(&l, self->ctx); + if (r < 0) + goto ERROR; + + // Ensure the repo is internalised + pakfire_repo_internalize(self, 0); + + FOR_REPO_SOLVABLES(self->repo, i, s) { + // Convert the solvable into an ID + id = pool_solvable2id(self->repo->pool, s); + + // Create a new package + r = pakfire_package_create_from_solvable(&pkg, self->pakfire, self, id); + if (r < 0) + goto ERROR; + + // Add the package to the list + r = pakfire_packagelist_add(l, pkg); + if (r < 0) + goto ERROR; + + pakfire_package_unref(pkg); + pkg = NULL; + } + + // Return the list + *list = pakfire_packagelist_ref(l); + +ERROR: + if (pkg) + pakfire_package_unref(pkg); + if (l) + pakfire_packagelist_unref(l); + + return r; +} + Id pakfire_repo_add_solvable(struct pakfire_repo* repo) { Id id = repo_add_solvable(repo->repo); @@ -535,6 +582,233 @@ struct pakfire_mirrorlist* pakfire_repo_get_mirrorlist(struct pakfire_repo* self return NULL; } +struct pakfire_repo_scan_ctx { + // Progress + struct pakfire_progress* progress; + + // Flags + enum pakfire_repo_scan_flags { + PAKFIRE_REPO_SCAN_SKIP_EXISTING = (1 << 0), + } flags; + + // Counter + unsigned int num_archives; + + // Packages + struct pakfire_packagelist* packages; +}; + +typedef int (*pakfire_repo_scan_callback)( + struct pakfire_repo* repo, struct pakfire_repo_scan_ctx* scan_ctx, FTSENT* entry); + +static int __pakfire_repo_scan(struct pakfire_repo* repo, + struct pakfire_repo_scan_ctx* scan_ctx, pakfire_repo_scan_callback callback) { + FTSENT* entry = NULL; + FTS* fts = NULL; + int r = 0; + + // Fetch the repository path + const char* path = pakfire_repo_get_path(repo); + if (!path) + return -EINVAL; + + char* paths[] = { + (char*)path, NULL, + }; + + // Open the path + fts = fts_open(paths, FTS_PHYSICAL|FTS_NOCHDIR|FTS_NOSTAT, NULL); + if (!fts) { + ERROR(repo->ctx, "Could not open %s: %m\n", path); + r = -errno; + goto ERROR; + } + + // Scan for everything + for (;;) { + entry = fts_read(fts); + if (!entry) + break; + + // We only care about actual files + if (!(entry->fts_info & FTS_F)) + continue; + + // Skip anything that doesn't have the correct file extension + if (!pakfire_path_match("**.pfm", entry->fts_path)) + continue; + + // Call the callback + r = callback(repo, scan_ctx, entry); + if (r) + goto ERROR; + } + +ERROR: + if (fts) + fts_close(fts); + + return r; +} + +static int __pakfire_repo_scan_count(struct pakfire_repo* repo, + struct pakfire_repo_scan_ctx* scan_ctx, FTSENT* entry) { + ++scan_ctx->num_archives; + + return 0; +} + +static int __pakfire_repo_scan_archive(struct pakfire_repo* repo, + struct pakfire_repo_scan_ctx* scan_ctx, FTSENT* entry) { + struct pakfire_archive* archive = NULL; + struct pakfire_package* package = NULL; + int r = 0; + + // Skip this package if it already exists + if (scan_ctx->flags & PAKFIRE_REPO_SCAN_SKIP_EXISTING) { + if (pakfire_packagelist_has_path(scan_ctx->packages, entry->fts_path)) { + goto DONE; + } + } + + // Open archive + r = pakfire_archive_open(&archive, repo->pakfire, entry->fts_path); + if (r < 0) + goto ERROR; + + // Import package into the repository + r = pakfire_archive_make_package(archive, repo, &package); + if (r < 0) + goto ERROR; + +DONE: + // Increment progress bar + pakfire_progress_increment(scan_ctx->progress, 1); + +ERROR: + if (package) + pakfire_package_unref(package); + if (archive) + pakfire_archive_unref(archive); + + return r; +} + +int pakfire_repo_scan(struct pakfire_repo* repo, int flags) { + int r; + + // Scan context + struct pakfire_repo_scan_ctx scan_ctx = { + .flags = flags, + }; + + // Fetch name + const char* name = pakfire_repo_get_name(repo); + + // Fetch path + const char* path = pakfire_repo_get_path(repo); + if (!path) + return -EINVAL; + + // Create progress indicator + r = pakfire_progress_create(&scan_ctx.progress, repo->ctx, + PAKFIRE_PROGRESS_SHOW_COUNTER|PAKFIRE_PROGRESS_SHOW_ELAPSED_TIME, NULL); + if (r) + goto ERROR; + + // Add title to progress + r = pakfire_progress_set_title(scan_ctx.progress, _("Scanning %s"), name); + if (r) + goto ERROR; + + // Load all existing packages + if (scan_ctx.flags & PAKFIRE_REPO_SCAN_SKIP_EXISTING) { + r = pakfire_repo_to_packagelist(repo, &scan_ctx.packages); + if (r < 0) + goto ERROR; + } + + // Start progress (so that we will let the user know something is happening if + // scanning for files takes a little bit longer...) + r = pakfire_progress_start(scan_ctx.progress, 0); + if (r < 0) + goto ERROR; + + // Count how many files we have + r = __pakfire_repo_scan(repo, &scan_ctx, __pakfire_repo_scan_count); + if (r < 0) + goto ERROR; + + // Update the progressbar + pakfire_progress_set_max_value(scan_ctx.progress, scan_ctx.num_archives); + + // Scan all packages + r = __pakfire_repo_scan(repo, &scan_ctx, __pakfire_repo_scan_archive); + if (r < 0) + goto ERROR; + + // Mark repository data as changed + pakfire_repo_has_changed(repo); + + // Finish the progress + r = pakfire_progress_finish(scan_ctx.progress); + if (r < 0) + goto ERROR; + + // Success + r = 0; + +ERROR: + if (scan_ctx.packages) + pakfire_packagelist_unref(scan_ctx.packages); + if (scan_ctx.progress) + pakfire_progress_unref(scan_ctx.progress); + + return r; +} + +static int pakfire_repo_sync_remove(struct pakfire_ctx* ctx, struct pakfire_package* pkg, void* data) { + int r; + + // Check if the package is available + r = pakfire_package_is_available(pkg); + switch (r) { + // The package is not available + case 0: + // Destroy the package + return pakfire_package_destroy(pkg); + + // The package is available + case 1: + return 0; + + // Error + default: + return r; + } + + return 0; +} + +static int pakfire_repo_sync(struct pakfire_repo* self) { + int r; + + // Log action + DEBUG(self->ctx, "Syncing %s...\n", pakfire_repo_get_name(self)); + + // Remove all packages that have disappeared + r = pakfire_repo_walk_packages(self, pakfire_repo_sync_remove, NULL, 0); + if (r < 0) + return r; + + // We need to find all new packages + r = pakfire_repo_scan(self, PAKFIRE_REPO_SCAN_SKIP_EXISTING); + if (r < 0) + return r; + + return 0; +} + static int pakfire_repo_download_database( struct pakfire_repo* repo, const char* filename, const char* path) { struct pakfire_xfer* xfer = NULL; @@ -641,6 +915,13 @@ static int pakfire_repo_read_database(struct pakfire_repo* self) { if (r < 0) goto ERROR; + // Perform a sync for all local repositories + if (pakfire_repo_is_local(self)) { + r = pakfire_repo_sync(self); + if (r < 0) + goto ERROR; + } + ERROR: if (f) fclose(f); @@ -757,14 +1038,6 @@ static int pakfire_repo_read_metadata(struct pakfire_repo* repo, const char* pat if (!json) { switch (errno) { case ENOENT: - // If the repository is local, we will just scan it - if (pakfire_repo_is_local(repo)) { - DEBUG(repo->ctx, "No metadata available on local repository." - " Falling back to scan...\n"); - - return pakfire_repo_scan(repo, 0); - } - // Log some message DEBUG(repo->ctx, "Repository metadata is not available\n"); @@ -1262,69 +1535,22 @@ int pakfire_repo_cmp(struct pakfire_repo* repo1, struct pakfire_repo* repo2) { return 1; else if (r1->priority < r2->priority) - return -1; - - return strcmp(r1->name, r2->name); -} - -int pakfire_repo_count(struct pakfire_repo* repo) { - Pool* pool = pakfire_get_solv_pool(repo->pakfire); - int cnt = 0; - - for (int i = 2; i < pool->nsolvables; i++) { - Solvable* s = pool->solvables + i; - if (s->repo && s->repo == repo->repo) - cnt++; - } - - return cnt; -} - -static int pakfire_repo_to_packagelist( - struct pakfire_repo* self, struct pakfire_packagelist** list) { - struct pakfire_packagelist* l = NULL; - struct pakfire_package* pkg = NULL; - Solvable* s = NULL; - Id id; - int i; - int r; - - // Create a new packagelist - r = pakfire_packagelist_create(&l, self->ctx); - if (r < 0) - goto ERROR; - - // Ensure the repo is internalised - pakfire_repo_internalize(self, 0); - - FOR_REPO_SOLVABLES(self->repo, i, s) { - // Convert the solvable into an ID - id = pool_solvable2id(self->repo->pool, s); - - // Create a new package - r = pakfire_package_create_from_solvable(&pkg, self->pakfire, self, id); - if (r < 0) - goto ERROR; - - // Add the package to the list - r = pakfire_packagelist_add(l, pkg); - if (r < 0) - goto ERROR; + return -1; - pakfire_package_unref(pkg); - pkg = NULL; - } + return strcmp(r1->name, r2->name); +} - // Return the list - *list = pakfire_packagelist_ref(l); +int pakfire_repo_count(struct pakfire_repo* repo) { + Pool* pool = pakfire_get_solv_pool(repo->pakfire); + int cnt = 0; -ERROR: - if (pkg) - pakfire_package_unref(pkg); - if (l) - pakfire_packagelist_unref(l); + for (int i = 2; i < pool->nsolvables; i++) { + Solvable* s = pool->solvables + i; + if (s->repo && s->repo == repo->repo) + cnt++; + } - return r; + return cnt; } void pakfire_repo_has_changed(struct pakfire_repo* repo) { @@ -1921,233 +2147,6 @@ int pakfire_repo_clean(struct pakfire_repo* repo, int flags) { return pakfire_rmtree(cache_path, 0); } -struct pakfire_repo_scan_ctx { - // Progress - struct pakfire_progress* progress; - - // Flags - enum pakfire_repo_scan_flags { - PAKFIRE_REPO_SCAN_SKIP_EXISTING = (1 << 0), - } flags; - - // Counter - unsigned int num_archives; - - // Packages - struct pakfire_packagelist* packages; -}; - -typedef int (*pakfire_repo_scan_callback)( - struct pakfire_repo* repo, struct pakfire_repo_scan_ctx* scan_ctx, FTSENT* entry); - -static int __pakfire_repo_scan(struct pakfire_repo* repo, - struct pakfire_repo_scan_ctx* scan_ctx, pakfire_repo_scan_callback callback) { - FTSENT* entry = NULL; - FTS* fts = NULL; - int r = 0; - - // Fetch the repository path - const char* path = pakfire_repo_get_path(repo); - if (!path) - return -EINVAL; - - char* paths[] = { - (char*)path, NULL, - }; - - // Open the path - fts = fts_open(paths, FTS_PHYSICAL|FTS_NOCHDIR|FTS_NOSTAT, NULL); - if (!fts) { - ERROR(repo->ctx, "Could not open %s: %m\n", path); - r = -errno; - goto ERROR; - } - - // Scan for everything - for (;;) { - entry = fts_read(fts); - if (!entry) - break; - - // We only care about actual files - if (!(entry->fts_info & FTS_F)) - continue; - - // Skip anything that doesn't have the correct file extension - if (!pakfire_path_match("**.pfm", entry->fts_path)) - continue; - - // Call the callback - r = callback(repo, scan_ctx, entry); - if (r) - goto ERROR; - } - -ERROR: - if (fts) - fts_close(fts); - - return r; -} - -static int __pakfire_repo_scan_count(struct pakfire_repo* repo, - struct pakfire_repo_scan_ctx* scan_ctx, FTSENT* entry) { - ++scan_ctx->num_archives; - - return 0; -} - -static int __pakfire_repo_scan_archive(struct pakfire_repo* repo, - struct pakfire_repo_scan_ctx* scan_ctx, FTSENT* entry) { - struct pakfire_archive* archive = NULL; - struct pakfire_package* package = NULL; - int r = 0; - - // Skip this package if it already exists - if (scan_ctx->flags & PAKFIRE_REPO_SCAN_SKIP_EXISTING) { - if (pakfire_packagelist_has_path(scan_ctx->packages, entry->fts_path)) { - goto DONE; - } - } - - // Open archive - r = pakfire_archive_open(&archive, repo->pakfire, entry->fts_path); - if (r < 0) - goto ERROR; - - // Import package into the repository - r = pakfire_archive_make_package(archive, repo, &package); - if (r < 0) - goto ERROR; - -DONE: - // Increment progress bar - pakfire_progress_increment(scan_ctx->progress, 1); - -ERROR: - if (package) - pakfire_package_unref(package); - if (archive) - pakfire_archive_unref(archive); - - return r; -} - -int pakfire_repo_scan(struct pakfire_repo* repo, int flags) { - int r; - - // Scan context - struct pakfire_repo_scan_ctx scan_ctx = { - .flags = flags, - }; - - // Fetch name - const char* name = pakfire_repo_get_name(repo); - - // Fetch path - const char* path = pakfire_repo_get_path(repo); - if (!path) - return -EINVAL; - - // Create progress indicator - r = pakfire_progress_create(&scan_ctx.progress, repo->ctx, - PAKFIRE_PROGRESS_SHOW_COUNTER|PAKFIRE_PROGRESS_SHOW_ELAPSED_TIME, NULL); - if (r) - goto ERROR; - - // Add title to progress - r = pakfire_progress_set_title(scan_ctx.progress, _("Scanning %s"), name); - if (r) - goto ERROR; - - // Load all existing packages - if (scan_ctx.flags & PAKFIRE_REPO_SCAN_SKIP_EXISTING) { - r = pakfire_repo_to_packagelist(repo, &scan_ctx.packages); - if (r < 0) - goto ERROR; - } - - // Start progress (so that we will let the user know something is happening if - // scanning for files takes a little bit longer...) - r = pakfire_progress_start(scan_ctx.progress, 0); - if (r < 0) - goto ERROR; - - // Count how many files we have - r = __pakfire_repo_scan(repo, &scan_ctx, __pakfire_repo_scan_count); - if (r < 0) - goto ERROR; - - // Update the progressbar - pakfire_progress_set_max_value(scan_ctx.progress, scan_ctx.num_archives); - - // Scan all packages - r = __pakfire_repo_scan(repo, &scan_ctx, __pakfire_repo_scan_archive); - if (r < 0) - goto ERROR; - - // Mark repository data as changed - pakfire_repo_has_changed(repo); - - // Finish the progress - r = pakfire_progress_finish(scan_ctx.progress); - if (r < 0) - goto ERROR; - - // Success - r = 0; - -ERROR: - if (scan_ctx.packages) - pakfire_packagelist_unref(scan_ctx.packages); - if (scan_ctx.progress) - pakfire_progress_unref(scan_ctx.progress); - - return r; -} - -static int pakfire_repo_sync_remove(struct pakfire_ctx* ctx, struct pakfire_package* pkg, void* data) { - int r; - - // Check if the package is available - r = pakfire_package_is_available(pkg); - switch (r) { - // The package is not available - case 0: - // Destroy the package - return pakfire_package_destroy(pkg); - - // The package is available - case 1: - return 0; - - // Error - default: - return r; - } - - return 0; -} - -static int pakfire_repo_sync(struct pakfire_repo* self) { - int r; - - // Log action - DEBUG(self->ctx, "Syncing %s...\n", pakfire_repo_get_name(self)); - - // Remove all packages that have disappeared - r = pakfire_repo_walk_packages(self, pakfire_repo_sync_remove, NULL, 0); - if (r < 0) - return r; - - // We need to find all new packages - r = pakfire_repo_scan(self, PAKFIRE_REPO_SCAN_SKIP_EXISTING); - if (r < 0) - return r; - - return 0; -} - int pakfire_repo_refresh(struct pakfire_repo* repo, int force) { char path[PATH_MAX]; int r; @@ -2195,13 +2194,6 @@ int pakfire_repo_refresh(struct pakfire_repo* repo, int force) { if (r < 0) return r; - // Perform a sync for all local repositories - if (pakfire_repo_is_local(repo)) { - r = pakfire_repo_sync(repo); - if (r < 0) - return r; - } - return 0; } -- 2.39.5