From: Michael Tremer Date: Wed, 17 Aug 2022 19:47:04 +0000 (+0000) Subject: compress: Create a unified extraction function X-Git-Tag: 0.9.28~464 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=79824416461dea0370d6132e3a892463ccb57461;p=pakfire.git compress: Create a unified extraction function Signed-off-by: Michael Tremer --- diff --git a/src/_pakfire/archive.c b/src/_pakfire/archive.c index 0765a790c..058a32975 100644 --- a/src/_pakfire/archive.c +++ b/src/_pakfire/archive.c @@ -147,14 +147,9 @@ static PyObject* Archive_sign(ArchiveObject* self, PyObject* args) { Py_RETURN_NONE; } -static PyObject* Archive_extract(ArchiveObject* self, PyObject* args) { - const char* prefix = NULL; - - if (!PyArg_ParseTuple(args, "|z", &prefix)) - return NULL; - +static PyObject* Archive_extract(ArchiveObject* self) { // Extract payload - int r = pakfire_archive_extract(self->archive, prefix); + int r = pakfire_archive_extract(self->archive); if (r) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -266,7 +261,7 @@ static struct PyMethodDef Archive_methods[] = { { "extract", (PyCFunction)Archive_extract, - METH_VARARGS, + METH_NOARGS, NULL }, { diff --git a/src/libpakfire/archive.c b/src/libpakfire/archive.c index 2730698f5..fca06e12c 100644 --- a/src/libpakfire/archive.c +++ b/src/libpakfire/archive.c @@ -41,6 +41,7 @@ #include #include +#include #include #include #include @@ -856,190 +857,41 @@ ERROR: return r; } -static int pakfire_archive_extraction_path(struct pakfire_archive* archive, - char* path, size_t length, const char* prefix) { - char buffer[PATH_MAX]; - - struct pakfire_package* pkg = pakfire_archive_get_package(archive); - if (!pkg) { - ERROR(archive->pakfire, "Could not fetch package from archive: %m\n"); - return 1; - } - - // Use a good default when no prefix is set - if (pakfire_package_is_source(pkg)) { - if (!prefix) - prefix = "/usr/src/packages"; - - pakfire_string_format(buffer, "%s/%s", prefix, pakfire_package_get_nevra(pkg)); - } else { - if (!prefix) - prefix = "/"; - - pakfire_string_set(buffer, prefix); - } - - // Always prepend the root path - __pakfire_make_path(archive->pakfire, path, length, buffer); - - // Cleanup - pakfire_package_unref(pkg); - - return 0; -} - -struct pakfire_archive_extractor { - struct archive* writer; - const char* path; - struct pakfire_progressbar* progressbar; -}; - -static int pakfire_archive_extract_progressbar( - struct pakfire_archive* archive, struct pakfire_progressbar** progressbar) { +PAKFIRE_EXPORT int pakfire_archive_extract(struct pakfire_archive* archive) { + struct pakfire_package* pkg = NULL; + const char* prefix = NULL; + struct archive* a = NULL; + struct archive* payload = NULL; + size_t size = 0; int r = 1; - struct pakfire_package* pkg = pakfire_archive_get_package(archive); + // Fetch package + pkg = pakfire_archive_get_package(archive); if (!pkg) goto ERROR; - r = pakfire_progressbar_create(progressbar, NULL); - if (r) - goto ERROR; - - // Add progressbar widgets - r = pakfire_progressbar_add_string(*progressbar, "%s", pakfire_package_get_nevra(pkg)); - if (r) - goto ERROR; - - r = pakfire_progressbar_add_bar(*progressbar); - if (r) - goto ERROR; - - r = pakfire_progressbar_add_percentage(*progressbar); - if (r) - goto ERROR; - - // Success - r = 0; - -ERROR: - pakfire_package_unref(pkg); - - return r; -} - -static int pakfire_archive_extract_entry(struct pakfire_archive* archive, - struct archive* a, struct archive_entry* entry, int flags, void* data) { - struct pakfire_archive_extractor* extractor = (struct pakfire_archive_extractor*)data; - - char buffer[PATH_MAX]; - int r = 1; - - // How much of the archive has been read? - size_t bytes_read = archive_filter_bytes(a, -1); - - // Update progress - if (extractor->progressbar) - pakfire_progressbar_update(extractor->progressbar, bytes_read); - - // Fetch the paths - const char* path = archive_entry_pathname(entry); - - // Prepend the prefix - if (extractor->path && *extractor->path) { - r = pakfire_path_join(buffer, extractor->path, path); - if (r < 0) - return r; - - archive_entry_set_pathname(entry, buffer); - - // Update hardlink destination - const char* link = archive_entry_hardlink(entry); - if (link) { - r = pakfire_path_join(buffer, extractor->path, link); - if (r < 0) - return r; - - archive_entry_set_hardlink(entry, buffer); - } - } - - // Create file & extract payload - r = archive_read_extract2(a, entry, extractor->writer); - switch (r) { - case ARCHIVE_OK: - break; - - case ARCHIVE_WARN: - ERROR(archive->pakfire, "%s\n", archive_error_string(extractor->writer)); - - // Pretend everything has been okay - r = ARCHIVE_OK; - break; - - case ARCHIVE_FATAL: - ERROR(archive->pakfire, "%s\n", archive_error_string(extractor->writer)); - return 1; - } - - return 0; -} - -PAKFIRE_EXPORT int pakfire_archive_extract(struct pakfire_archive* archive, const char* prefix) { - struct pakfire_progressbar* progressbar = NULL; - char path[PATH_MAX]; - struct archive* a = NULL; - struct archive* payload = NULL; - struct archive* writer = NULL; - size_t size; + // Fetch NEVRA + const char* nevra = pakfire_package_get_nevra(pkg); - int r = pakfire_archive_extraction_path(archive, path, sizeof(path) - 1, prefix); - if (r) - goto ERROR; + DEBUG(archive->pakfire, "Extracting %s\n", archive->path); - DEBUG(archive->pakfire, "Extracting %s to %s\n", archive->path, path); - - // Create a progressbar - r = pakfire_archive_extract_progressbar(archive, &progressbar); - if (r) - goto ERROR; + // Set prefix for source packages + if (pakfire_package_is_source(pkg)) + prefix = "/usr/src/packages"; // Open payload payload = pakfire_archive_open_payload(archive, &a, &size); if (!payload) goto ERROR; - // Allocate writer - writer = pakfire_make_archive_disk_writer(archive->pakfire, 1); - if (!writer) + // Extract + r = pakfire_extract(archive->pakfire, payload, size, prefix, nevra, 0); + if (r) goto ERROR; - struct pakfire_archive_extractor extractor = { - .writer = writer, - .path = path, - .progressbar = progressbar, - }; - - // Start the progressbar - if (progressbar) { - r = pakfire_progressbar_start(progressbar, size); - if (r) - goto ERROR; - } - - // Extract everything - r = pakfire_archive_walk_entries(archive, payload, - pakfire_archive_extract_entry, 0, &extractor, NULL); - ERROR: - // Finish progressbar - if (progressbar) - pakfire_progressbar_finish(progressbar); - - if (progressbar) - pakfire_progressbar_unref(progressbar); - if (writer) - archive_write_free(writer); + if (pkg) + pakfire_package_unref(pkg); if (payload) archive_read_free(payload); if (a) diff --git a/src/libpakfire/compress.c b/src/libpakfire/compress.c index 91c81d779..3da28eab3 100644 --- a/src/libpakfire/compress.c +++ b/src/libpakfire/compress.c @@ -23,10 +23,14 @@ #include #include +#include #include #include #include +#include +#include +#include // Read up to N bytes for analyze the magic #define MAX_MAGIC_LENGTH 6 @@ -511,3 +515,192 @@ ERROR: return NULL; } + +// Common extraction + +struct pakfire_extract { + // Reference to Pakfire + struct pakfire* pakfire; + + // The archive to extract + struct archive* archive; + + // Prepend this prefix + char prefix[PATH_MAX]; + + // The writer + struct archive* writer; + + // The progressbar + struct pakfire_progressbar* progressbar; +}; + +static int pakfire_extract_progressbar_create(struct pakfire_progressbar** progressbar, + const char* message, int flags) { + int r; + + // Create the progressbar + r = pakfire_progressbar_create(progressbar, NULL); + if (r) + return r; + + // Add message + if (message) { + r = pakfire_progressbar_add_string(*progressbar, "%s", message); + if (r) + return r; + } + + // Add bar + r = pakfire_progressbar_add_bar(*progressbar); + if (r) + return r; + + // Add percentage + r = pakfire_progressbar_add_percentage(*progressbar); + if (r) + return r; + + // Success + return 0; +} + +static void pakfire_extract_progress(void* p) { + struct pakfire_extract* data = (struct pakfire_extract*)p; + + // Fetch how many bytes have been read + const size_t position = archive_filter_bytes(data->archive, -1); + + // Update progressbar + pakfire_progressbar_update(data->progressbar, position); +} + +static int __pakfire_extract_entry(struct pakfire* pakfire, struct pakfire_extract* data, + struct archive_entry* entry) { + char buffer[PATH_MAX]; + int r; + + // Fetch path + const char* path = archive_entry_pathname(entry); + + DEBUG(pakfire, "Extracting %s\n", path); + + // Prepend the prefix + if (*data->prefix) { + // Compose file path + r = pakfire_path_join(buffer, data->prefix, path); + if (r < 0) { + ERROR(pakfire, "Could not compose file path: %m\n"); + return r; + } + + // Set file path + archive_entry_set_pathname(entry, buffer); + + // Update hardlink destination + const char* link = archive_entry_hardlink(entry); + if (link) { + r = pakfire_path_join(buffer, data->prefix, link); + if (r < 0) { + ERROR(pakfire, "Could not compose hardlink path: %m\n"); + return r; + } + + // Set hardlink path + archive_entry_set_hardlink(entry, buffer); + } + } + + // Create file & extract payload + r = archive_read_extract2(data->archive, entry, data->writer); + switch (r) { + case ARCHIVE_OK: + r = 0; + break; + + case ARCHIVE_WARN: + ERROR(pakfire, "%s\n", archive_error_string(data->writer)); + + // Pretend everything has been okay + r = 0; + break; + + case ARCHIVE_FATAL: + ERROR(pakfire, "%s\n", archive_error_string(data->writer)); + r = 1; + break; + } + + return r; +} + +int pakfire_extract(struct pakfire* pakfire, struct archive* archive, + size_t size, const char* prefix, const char* message, int flags) { + int r = 1; + + struct pakfire_extract data = { + .pakfire = pakfire, + .archive = archive, + }; + + // Set prefix (including pakfire path) + pakfire_make_path(pakfire, data.prefix, prefix); + + // Allocate writer + data.writer = pakfire_make_archive_disk_writer(pakfire, 1); + if (!data.writer) { + ERROR(pakfire, "Could not create disk writer: %m\n"); + goto ERROR; + } + + // Create the progressbar + r = pakfire_extract_progressbar_create(&data.progressbar, message, flags); + if (r) + goto ERROR; + + // Register progress callback + if (data.progressbar) + archive_read_extract_set_progress_callback(data.archive, + pakfire_extract_progress, &data); + + // Start progressbar + if (data.progressbar) + pakfire_progressbar_start(data.progressbar, size); + + struct archive_entry* entry = NULL; + + // Walk through the archive + while (1) { + r = archive_read_next_header(archive, &entry); + + // End when we have reached the end of the archive + if (r == ARCHIVE_EOF) { + r = 0; + break; + } + + // Raise any other errors + else if (r) { + ERROR(pakfire, "Could not read next header: %s\n", + archive_error_string(archive)); + goto ERROR; + } + + // Extract the entry + r = __pakfire_extract_entry(pakfire, &data, entry); + if (r) + goto ERROR; + } + + // Finish the progressbar + if (data.progressbar) + pakfire_progressbar_finish(data.progressbar); + +ERROR: + if (data.progressbar) + pakfire_progressbar_unref(data.progressbar); + if (data.writer) + archive_write_free(data.writer); + + return r; +} diff --git a/src/libpakfire/include/pakfire/archive.h b/src/libpakfire/include/pakfire/archive.h index 26c79a139..c12eebf3c 100644 --- a/src/libpakfire/include/pakfire/archive.h +++ b/src/libpakfire/include/pakfire/archive.h @@ -50,7 +50,7 @@ char* pakfire_archive_get(struct pakfire_archive* archive, const char* namespace int pakfire_archive_read(struct pakfire_archive* archive, const char* filename, char** data, size_t* data_size); -int pakfire_archive_extract(struct pakfire_archive* archive, const char* prefix); +int pakfire_archive_extract(struct pakfire_archive* archive); const char* pakfire_archive_get_path(struct pakfire_archive* archive); diff --git a/src/libpakfire/include/pakfire/compress.h b/src/libpakfire/include/pakfire/compress.h index 633ad2997..eef3fa219 100644 --- a/src/libpakfire/include/pakfire/compress.h +++ b/src/libpakfire/include/pakfire/compress.h @@ -23,6 +23,10 @@ #ifdef PAKFIRE_PRIVATE +#include + +#include + // Automatically detect FILE* pakfire_xfopen(FILE* f, const char* mode); @@ -32,6 +36,10 @@ FILE* pakfire_xzfopen(FILE* f, const char* mode); // ZSTD FILE* pakfire_zstdfopen(FILE* f, const char* mode); +// Extract +int pakfire_extract(struct pakfire* pakfire, struct archive* archive, + size_t size, const char* prefix, const char* message, int flags); + #endif #endif /* PAKFIRE_COMPRESS_H */ diff --git a/src/libpakfire/snapshot.c b/src/libpakfire/snapshot.c index 6ee8e2aff..13fd4e4bc 100644 --- a/src/libpakfire/snapshot.c +++ b/src/libpakfire/snapshot.c @@ -25,51 +25,16 @@ #include +#include #include #include #include #include #include -#include #include #include #include -static int pakfire_snapshot_progressbar( - struct pakfire_progressbar** progressbar, const char* message) { - int r = pakfire_progressbar_create(progressbar, NULL); - if (r) - return r; - - // Add message - r = pakfire_progressbar_add_string(*progressbar, "%s", message); - if (r) - goto ERROR; - - // Add bar - r = pakfire_progressbar_add_bar(*progressbar); - if (r) - goto ERROR; - - // Add throughput - r = pakfire_progressbar_add_transfer_speed(*progressbar); - if (r) - goto ERROR; - - // Add ETA - r = pakfire_progressbar_add_eta(*progressbar); - if (r) - goto ERROR; - - return 0; - -ERROR: - pakfire_progressbar_unref(*progressbar); - *progressbar = NULL; - - return 1; -} - static struct archive* pakfire_snapshot_create_archive( struct pakfire* pakfire, const char* path) { struct archive* a = archive_write_new(); @@ -233,8 +198,6 @@ ERROR: } static int pakfire_snapshot_extract(struct pakfire* pakfire, const char* path) { - struct pakfire_progressbar* progressbar = NULL; - char buffer[PATH_MAX]; struct stat st; int r = 1; @@ -251,92 +214,31 @@ static int pakfire_snapshot_extract(struct pakfire* pakfire, const char* path) { return 1; } - struct archive* a = archive_read_new(); - if (!a) + struct archive* archive = archive_read_new(); + if (!archive) return 1; // All snapshots are tarballs - archive_read_support_format_tar(a); + archive_read_support_format_tar(archive); // And they are compressed using ZSTD - archive_read_support_filter_zstd(a); - - struct archive* disk = pakfire_make_archive_disk_writer(pakfire, 1); - if (!disk) - goto ERROR; + archive_read_support_filter_zstd(archive); // Open the given file for reading - r = archive_read_open_filename(a, path, 64 * 1024); + r = archive_read_open_filename(archive, path, 64 * 1024); if (r) { - ERROR(pakfire, "Could not open archive: %s\n", archive_error_string(a)); + ERROR(pakfire, "Could not open archive: %s\n", archive_error_string(archive)); goto ERROR; } - // Make progressbar - r = pakfire_snapshot_progressbar(&progressbar, _("Extracting snapshot...")); - if (r) - goto ERROR; - - r = pakfire_progressbar_start(progressbar, st.st_size); + // Extract snapshot + r = pakfire_extract(pakfire, archive, st.st_size, NULL, _("Extracting snapshot..."), 0); if (r) goto ERROR; - struct archive_entry* e = NULL; - while (1) { - r = archive_read_next_header(a, &e); - if (r == ARCHIVE_EOF) - break; - else if (r) { - ERROR(pakfire, "Could not read header: %s\n", archive_error_string(a)); - goto ERROR; - } - - // How much of the archive has been read? - size_t bytes_read = archive_filter_bytes(a, -1); - - // Update progress - pakfire_progressbar_update(progressbar, bytes_read); - - const char* _path = archive_entry_pathname(e); - - // Update path - r = pakfire_make_path(pakfire, buffer, _path); - if (r < 0) - goto ERROR; - - archive_entry_set_pathname(e, buffer); - - // Update hardlink destination - const char* link = archive_entry_hardlink(e); - if (link) { - r = pakfire_make_path(pakfire, buffer, link); - if (r < 0) - goto ERROR; - - archive_entry_set_hardlink(e, buffer); - } - - // Extract the file - r = archive_read_extract2(a, e, disk); - if (r) { - ERROR(pakfire, "Could not extract %s: %s\n", - archive_entry_pathname(e), archive_error_string(a)); - goto ERROR; - } - } - - // Finish the progressbar - pakfire_progressbar_finish(progressbar); - - // Success - r = 0; - ERROR: - if (progressbar) - pakfire_progressbar_unref(progressbar); - if (disk) - archive_write_free(disk); - archive_read_free(a); + if (archive) + archive_read_free(archive); return r; } diff --git a/src/libpakfire/transaction.c b/src/libpakfire/transaction.c index e931268c3..373efe451 100644 --- a/src/libpakfire/transaction.c +++ b/src/libpakfire/transaction.c @@ -748,8 +748,8 @@ static int pakfire_transaction_extract(struct pakfire_transaction* transaction, // Update status pakfire_transaction_status(transaction, _("Installing %s..."), nevra); - // Extract payload to the root of the Pakfire instance - int r = pakfire_archive_extract(archive, NULL); + // Extract payload + int r = pakfire_archive_extract(archive); if (r) { ERROR(transaction->pakfire, "Could not extract package %s: %m\n", nevra);