From 9c29eacbe52f6ba80dfa4a7220cd9d7804296fd3 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sat, 8 Feb 2025 13:01:07 +0000 Subject: [PATCH] compress: Drop old compression code This became a little bit messy with so many options and therefore I replaced this with the new archive writer. Signed-off-by: Michael Tremer --- Makefile.am | 2 - src/pakfire/archive.c | 1 - src/pakfire/compress.c | 414 ----------------------------------------- src/pakfire/compress.h | 49 ----- 4 files changed, 466 deletions(-) delete mode 100644 src/pakfire/compress.c delete mode 100644 src/pakfire/compress.h diff --git a/Makefile.am b/Makefile.am index c2c6b3d4..5e16d956 100644 --- a/Makefile.am +++ b/Makefile.am @@ -196,8 +196,6 @@ libpakfire_la_SOURCES = \ src/pakfire/buildservice.h \ src/pakfire/cgroup.c \ src/pakfire/cgroup.h \ - src/pakfire/compress.c \ - src/pakfire/compress.h \ src/pakfire/config.c \ src/pakfire/config.h \ src/pakfire/constants.h \ diff --git a/src/pakfire/archive.c b/src/pakfire/archive.c index 2513635a..5df266cd 100644 --- a/src/pakfire/archive.c +++ b/src/pakfire/archive.c @@ -34,7 +34,6 @@ #include #include -#include #include #include #include diff --git a/src/pakfire/compress.c b/src/pakfire/compress.c deleted file mode 100644 index f070f588..00000000 --- a/src/pakfire/compress.c +++ /dev/null @@ -1,414 +0,0 @@ -/*############################################################################# -# # -# Pakfire - The IPFire package management system # -# Copyright (C) 2021 Pakfire development team # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -#############################################################################*/ - -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Common compression - -struct pakfire_compress { - // Reference to context - struct pakfire_ctx* ctx; - - // Reference to Pakfire - struct pakfire* pakfire; - - // Flags - int flags; - - // The archive to write to - struct archive* archive; - - // Resolver for hardlinks - struct archive_entry_linkresolver* linkresolver; - - // The filelist of all files to write - struct pakfire_filelist* filelist; - - // The progress indicator - struct pakfire_progress* progress; - - // Checksums to write to the archive - int checksum_types; -}; - -static int pakfire_copy_data_from_file(struct pakfire_ctx* ctx, - struct archive* archive, FILE* f) { - ssize_t bytes_written = 0; - ssize_t bytes_read = 0; - char buffer[64 * 1024]; - - // Loop through the entire length of the file - while (!feof(f)) { - // Read a block from file - bytes_read = fread(buffer, 1, sizeof(buffer), f); - - // Check if any error occured - if (ferror(f)) { - ERROR(ctx, "Read error: %m\n"); - return -errno; - } - - // Write the block to the archive - bytes_written = archive_write_data(archive, buffer, bytes_read); - if (bytes_written < bytes_read) { - ERROR(ctx, "Write error: %s\n", archive_error_string(archive)); - return -errno; - } - } - - return 0; -} - -static int __pakfire_compress_entry(struct pakfire_ctx* ctx, struct pakfire_file* file, - struct pakfire_compress* data, struct archive_entry* entry) { - FILE* f = NULL; - int r; - - const char* path = archive_entry_pathname(entry); - - // Remove any leading slahes - while (*path == '/') - path++; - - archive_entry_set_pathname(entry, path); - - // Write the header - r = archive_write_header(data->archive, entry); - if (r) { - ERROR(data->ctx, "Error writing file header: %s\n", - archive_error_string(data->archive)); - goto ERROR; - } - - // Copy the data if there is any - if (archive_entry_size(entry)) { - // Open the file - f = pakfire_file_fopen(file, "r"); - if (!f) { - r = -errno; - goto ERROR; - } - - // Copy the payload into the archive - r = pakfire_copy_data_from_file(data->ctx, data->archive, f); - if (r) - goto ERROR; - } - - // Write trailer - r = archive_write_finish_entry(data->archive); - if (r) - goto ERROR; - -ERROR: - if (f) - fclose(f); - - return r; -} - -static int __pakfire_compress(struct pakfire_ctx* ctx, struct pakfire_file* file, void* p) { - struct archive_entry* entry = NULL; - struct archive_entry* sparse_entry = NULL; - int r = 1; - - struct pakfire_compress* data = (struct pakfire_compress*)p; - - // Fetch the file size - const size_t size = pakfire_file_get_size(file); - - // Generate file metadata into an archive entry - entry = pakfire_file_archive_entry(file, data->checksum_types); - if (!entry) { - r = -errno; - goto ERROR; - } - - // Perform search for hardlinks - archive_entry_linkify(data->linkresolver, &entry, &sparse_entry); - - // Write the main entry - if (entry) { - r = __pakfire_compress_entry(ctx, file, data, entry); - if (r) - goto ERROR; - } - - // Write the sparse entry - if (sparse_entry) { - r = __pakfire_compress_entry(ctx, file, data, sparse_entry); - if (r) - goto ERROR; - } - - // Query the link resolver for any more entries - for (;;) { - // Free the entry - if (entry) { - archive_entry_free(entry); - entry = NULL; - } - - // Free the sparse entry - if (sparse_entry) { - archive_entry_free(sparse_entry); - sparse_entry = NULL; - } - - // Fetch the next entry - archive_entry_linkify(data->linkresolver, &entry, &sparse_entry); - if (!entry) - break; - - // Write the entry to the archive - r = __pakfire_compress_entry(ctx, file, data, entry); - if (r) - goto ERROR; - } - - // Update the progress - pakfire_progress_increment(data->progress, size); - -ERROR: - if (entry) - archive_entry_free(entry); - if (sparse_entry) - archive_entry_free(sparse_entry); - - return r; -} - -int pakfire_compress(struct pakfire* pakfire, struct archive* archive, - struct pakfire_filelist* filelist, const char* message, int flags, - enum pakfire_hash_type checksum_types) { - int progress_flags = - PAKFIRE_PROGRESS_SHOW_PERCENTAGE | - PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED; - int r = 1; - - struct pakfire_ctx* ctx = pakfire_ctx(pakfire); - - struct pakfire_compress data = { - .ctx = ctx, - .pakfire = pakfire, - .archive = archive, - .filelist = filelist, - .flags = flags, - .checksum_types = checksum_types, - }; - - // Should we show a progress bar? - if (flags & PAKFIRE_COMPRESS_NO_PROGRESS) - progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS; - - // Should we show throughput? - if (flags & PAKFIRE_COMPRESS_SHOW_THROUGHPUT) - progress_flags |= PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED; - - // Fetch the length of the filelist - const size_t size = pakfire_filelist_total_size(filelist); - - // Create the progress indicator - r = pakfire_progress_create(&data.progress, ctx, progress_flags, NULL); - if (r) - goto ERROR; - - // Set progress title - r = pakfire_progress_set_title(data.progress, "%s", message); - if (r) - goto ERROR; - - // Start progress - r = pakfire_progress_start(data.progress, size); - if (r) - goto ERROR; - - // Setup the link resolver - data.linkresolver = archive_entry_linkresolver_new(); - if (!data.linkresolver) { - ERROR(ctx, "Could not setup link resolver: m\n"); - goto ERROR; - } - - // Set the link resolver strategy - archive_entry_linkresolver_set_strategy(data.linkresolver, archive_format(archive)); - - // Walk through the entire filelist - r = pakfire_filelist_walk(filelist, __pakfire_compress, &data, 0, NULL); - if (r) - goto ERROR; - - // Finish the progress - if (data.progress) - pakfire_progress_finish(data.progress); - -ERROR: - if (data.progress) - pakfire_progress_unref(data.progress); - if (data.linkresolver) - archive_entry_linkresolver_free(data.linkresolver); - if (ctx) - pakfire_ctx_unref(ctx); - - return r; -} - -int pakfire_compress_create_archive(struct pakfire* pakfire, struct archive** archive, - FILE* f, const enum pakfire_compressions compression, const unsigned int level) { - struct archive* a = NULL; - char value[16]; - int r; - - struct pakfire_ctx* ctx = pakfire_ctx(pakfire); - - // Open a new archive - a = archive_write_new(); - if (!a) { - ERROR(ctx, "archive_write_new() failed\n"); - r = 1; - goto ERROR; - } - - // Use the PAX format - r = archive_write_set_format_pax(a); - if (r) { - ERROR(ctx, "Could not set format to PAX: %s\n", archive_error_string(a)); - goto ERROR; - } - - // Store any extended attributes in the SCHILY headers - r = archive_write_set_format_option(a, "pax", "xattrheader", "SCHILY"); - if (r) { - ERROR(ctx, "Could not set xattrheader option: %s\n", archive_error_string(a)); - goto ERROR; - } - - switch (compression) { - case PAKFIRE_COMPRESS_ZSTD: - // Enable Zstd - r = archive_write_add_filter_zstd(a); - if (r) { - ERROR(ctx, "Could not enable Zstandard compression: %s\n", - archive_error_string(a)); - goto ERROR; - } - - // Do not pad the last block - archive_write_set_bytes_in_last_block(a, 1); - - // Set compression level - if (level) { - r = pakfire_string_format(value, "%u", level); - if (r) - goto ERROR; - - r = archive_write_set_filter_option(a, NULL, "compression-level", value); - if (r) { - ERROR(ctx, "Could not set Zstandard compression level: %s\n", - archive_error_string(a)); - goto ERROR; - } - } - -#if ARCHIVE_VERSION_NUMBER >= 3007002 - // Enable long mode (supported from libarchive >= 3.7.2) - r = archive_write_set_filter_option(a, NULL, "long", "31"); - if (r) { - ERROR(ctx, "Failed to enable Zstandard long mode: %s\n", - archive_error_string(a)); - goto ERROR; - } -#endif /* ARCHIVE_VERSION_NUMBER >= 3007002 */ - -#if ARCHIVE_VERSION_NUMBER >= 3006000 - // Fetch numbers of processors - long processors = sysconf(_SC_NPROCESSORS_ONLN); - - if (processors > 1) { - r = pakfire_string_format(value, "%ld", processors); - if (r) { - ERROR(ctx, "Could not format threads: %m\n"); - goto ERROR; - } - - // Try using multiple threads - r = archive_write_set_filter_option(a, NULL, "threads", value); - if (r) { - ERROR(ctx, "Could not enable %ld threads for compression: %s\n", - processors, archive_error_string(a)); - goto ERROR; - } - } -#endif - break; - - case PAKFIRE_COMPRESS_GZIP: - // Enable gzip - r = archive_write_add_filter_gzip(a); - if (r) { - ERROR(ctx, "Could not enable gzip compression: %s\n", - archive_error_string(a)); - goto ERROR; - } - break; - } - - // Write archive to f - if (f) { - r = archive_write_open_FILE(a, f); - if (r) { - ERROR(ctx, "archive_write_open_FILE() failed: %s\n", archive_error_string(a)); - goto ERROR; - } - } - - // Success - *archive = a; - - if (ctx) - pakfire_ctx_unref(ctx); - - return 0; - -ERROR: - if (a) - archive_write_free(a); - - if (ctx) - pakfire_ctx_unref(ctx); - - return r; -} diff --git a/src/pakfire/compress.h b/src/pakfire/compress.h deleted file mode 100644 index 2d1797b7..00000000 --- a/src/pakfire/compress.h +++ /dev/null @@ -1,49 +0,0 @@ -/*############################################################################# -# # -# Pakfire - The IPFire package management system # -# Copyright (C) 2021 Pakfire development team # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -#############################################################################*/ - -#ifndef PAKFIRE_COMPRESS_H -#define PAKFIRE_COMPRESS_H - -#include - -#include -#include -#include -#include - -// Algorithms -enum pakfire_compressions { - PAKFIRE_COMPRESS_ZSTD, - PAKFIRE_COMPRESS_GZIP, -}; - -// Compress -enum pakfire_compress_flags { - PAKFIRE_COMPRESS_NO_PROGRESS = (1 << 1), - PAKFIRE_COMPRESS_SHOW_THROUGHPUT = (1 << 2), -}; - -int pakfire_compress(struct pakfire* pakfire, struct archive* archive, - struct pakfire_filelist* filelist, const char* message, int flags, enum pakfire_hash_type checksum_types); - -int pakfire_compress_create_archive(struct pakfire* pakfire, struct archive** archive, - FILE* f, const enum pakfire_compressions compression, const unsigned int level); - -#endif /* PAKFIRE_COMPRESS_H */ -- 2.39.5