From: Michael Tremer Date: Sat, 8 Feb 2025 10:15:03 +0000 (+0000) Subject: xfopen: Move transparent compression/decompression functions into a separate file X-Git-Tag: 0.9.30~83 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=7436ea845c99cee463cab532a93b228d7a3323cb;p=pakfire.git xfopen: Move transparent compression/decompression functions into a separate file Signed-off-by: Michael Tremer --- diff --git a/.gitignore b/.gitignore index 8dd48191..7e949870 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ /tests/libpakfire/archive /tests/libpakfire/build /tests/libpakfire/cgroup -/tests/libpakfire/compress /tests/libpakfire/config /tests/libpakfire/daemon /tests/libpakfire/db @@ -42,6 +41,7 @@ /tests/libpakfire/string /tests/libpakfire/util /tests/libpakfire/xfer +/tests/libpakfire/xfopen /tests/parser/test /tests/stub/root /tmp diff --git a/Makefile.am b/Makefile.am index 6813e47c..85ed1a6a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -301,7 +301,9 @@ libpakfire_la_SOURCES = \ src/pakfire/util.c \ src/pakfire/util.h \ src/pakfire/xfer.c \ - src/pakfire/xfer.h + src/pakfire/xfer.h \ + src/pakfire/xfopen.c \ + src/pakfire/xfopen.h libpakfire_la_CFLAGS = \ $(AM_CFLAGS) \ @@ -572,7 +574,6 @@ check_PROGRAMS += \ tests/libpakfire/archive \ tests/libpakfire/build \ tests/libpakfire/cgroup \ - tests/libpakfire/compress \ tests/libpakfire/config \ tests/libpakfire/daemon \ tests/libpakfire/db \ @@ -594,7 +595,8 @@ check_PROGRAMS += \ tests/libpakfire/repo \ tests/libpakfire/string \ tests/libpakfire/util \ - tests/libpakfire/xfer + tests/libpakfire/xfer \ + tests/libpakfire/xfopen dist_tests_libpakfire_main_SOURCES = \ tests/libpakfire/main.c @@ -671,21 +673,6 @@ tests_libpakfire_cgroup_LDFLAGS = \ tests_libpakfire_cgroup_LDADD = \ $(TESTSUITE_LDADD) -dist_tests_libpakfire_compress_SOURCES = \ - tests/libpakfire/compress.c - -tests_libpakfire_compress_CPPFLAGS = \ - $(TESTSUITE_CPPFLAGS) - -tests_libpakfire_compress_CFLAGS = \ - $(TESTSUITE_CFLAGS) - -tests_libpakfire_compress_LDFLAGS = \ - $(TESTSUITE_LDFLAGS) - -tests_libpakfire_compress_LDADD = \ - $(TESTSUITE_LDADD) - dist_tests_libpakfire_config_SOURCES = \ tests/libpakfire/config.c @@ -1016,6 +1003,21 @@ tests_libpakfire_xfer_LDFLAGS = \ tests_libpakfire_xfer_LDADD = \ $(TESTSUITE_LDADD) +dist_tests_libpakfire_xfopen_SOURCES = \ + tests/libpakfire/xfopen.c + +tests_libpakfire_xfopen_CPPFLAGS = \ + $(TESTSUITE_CPPFLAGS) + +tests_libpakfire_xfopen_CFLAGS = \ + $(TESTSUITE_CFLAGS) + +tests_libpakfire_xfopen_LDFLAGS = \ + $(TESTSUITE_LDFLAGS) + +tests_libpakfire_xfopen_LDADD = \ + $(TESTSUITE_LDADD) + # ------------------------------------------------------------------------------ noinst_PROGRAMS += \ diff --git a/src/pakfire/compress.c b/src/pakfire/compress.c index f51134a9..f070f588 100644 --- a/src/pakfire/compress.c +++ b/src/pakfire/compress.c @@ -24,9 +24,6 @@ #include #include -#include -#include -#include #include #include @@ -39,631 +36,6 @@ #include #include -// Read up to N bytes for analyze the magic -#define MAX_MAGIC_LENGTH 6 - -// Compression/Decompression buffer size -#define BUFFER_SIZE 64 * 1024 - -// Settings for XZ -#define XZ_COMPRESSION_LEVEL 6 - -// Settings for ZSTD -#define ZSTD_COMPRESSION_LEVEL 15 - -const struct compressor { - char magic[MAX_MAGIC_LENGTH]; - size_t magic_length; - FILE* (*open)(FILE* f, const char* mode); -} compressors[] = { - // Gzip - { { 0x1f, 0x8b }, 2, pakfire_gzfopen, }, - // XZ - { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, }, - // ZSTD - { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, }, - // End - { "", 0, NULL, }, -}; - -// Try to guess the compression -FILE* pakfire_xfopen(FILE* f, const char* mode) { - char buffer[MAX_MAGIC_LENGTH]; - - if (!f) { - errno = EBADFD; - return NULL; - } - - if (!mode) { - errno = EINVAL; - return NULL; - } - - // This only works for reading files - if (*mode != 'r') { - errno = ENOTSUP; - return NULL; - } - - fpos_t pos; - - // Store the position - int r = fgetpos(f, &pos); - if (r < 0) - return NULL; - - // Read a couple of bytes - size_t bytes_read = fread(buffer, 1, sizeof(buffer), f); - - // Reset position - r = fsetpos(f, &pos); - if (r < 0) - return NULL; - - // Check if we could read anything - if (!bytes_read || bytes_read < sizeof(buffer)) - return f; - - // Analyze magic - for (const struct compressor* c = compressors; c->open; c++) { - // Check if we have read enough data - if (bytes_read < c->magic_length) - continue; - - // Compare the magic value - r = memcmp(c->magic, buffer, c->magic_length); - if (r) - continue; - - // We found a match! - return c->open(f, mode); - } - - // Nothing seems to match - errno = ENOTSUP; - return f; -} - -static int parse_level(int* level, const char* mode) { - unsigned long value = 0; - char* p = NULL; - - // We must have mode - if (!mode || *mode != 'w') { - errno = EINVAL; - return -1; - } - - // Skip the w - mode++; - - // Do nothing if we have reached the end of the string - if (!*mode) - return 0; - - // Try parsing a number - value = strtoul(mode, &p, 10); - - // Break on error - if (value == ULONG_MAX) - return -1; - - // Check if we have parsed the entire string - if (p && *p) { - errno = EINVAL; - return -1; - } - - // Otherwise we return the value and return - *level = value; - return 0; -} - -/* - gzip -*/ - -static ssize_t gz_read(void* cookie, char* buffer, size_t size) { - return gzread((gzFile)cookie, buffer, size); -} - -static ssize_t gz_write(void* cookie, const char* buffer, size_t size) { - return gzwrite((gzFile)cookie, buffer, size); -} - -static int gz_close(void* cookie) { - return gzclose((gzFile)cookie); -} - -FILE* pakfire_gzfopen(FILE* f, const char* mode) { - gzFile cookie = NULL; - int fd = -EBADF; - - // Check for a valid file descriptor - if (!f) { - errno = EBADF; - return NULL; - } - - // Fetch the file descriptor - fd = fileno(f); - if (fd < 0) { - errno = EBADF; - return NULL; - } - - // Duplicate the file descriptor to pass it to gzip - fd = dup(fd); - if (fd < 0) - return NULL; - - // Close the original file handle - fclose(f); - - // Open the stream - cookie = gzdopen(fd, mode); - if (!cookie) - return NULL; - - // Increase the buffer size for faster reading - gzbuffer(cookie, 128 * 1024); - - static cookie_io_functions_t gz_functions = { - .read = gz_read, - .write = gz_write, - .close = gz_close, - }; - - return fopencookie(cookie, mode, gz_functions); -} - -struct xz_cookie { - FILE* f; - char mode; - lzma_stream stream; - int done; - - uint8_t buffer[BUFFER_SIZE]; -}; - -static ssize_t xz_read(void* data, char* buffer, size_t size) { - struct xz_cookie* cookie = (struct xz_cookie*)data; - if (!cookie) - return -1; - - // Do not read when mode is "w" - if (cookie->mode == 'w') - return -1; - - lzma_action action = LZMA_RUN; - - // Set output to allocated buffer - cookie->stream.next_out = (uint8_t *)buffer; - cookie->stream.avail_out = size; - - while (1) { - // Read something when the input buffer is empty - if (cookie->stream.avail_in == 0) { - cookie->stream.next_in = cookie->buffer; - cookie->stream.avail_in = fread(cookie->buffer, - 1, sizeof(cookie->buffer), cookie->f); - - // Break if the input file could not be read - if (ferror(cookie->f)) - return -1; - - // Finish after we have reached the end of the input file - if (feof(cookie->f)) - cookie->done = 1; - } - - lzma_ret ret = lzma_code(&cookie->stream, action); - - // If the stream has ended, we just send the - // remaining output and mark that we are done. - if (ret == LZMA_STREAM_END) { - cookie->done = 1; - return size - cookie->stream.avail_out; - } - - // Break on all other unexpected errors - if (ret != LZMA_OK) - return -1; - - // When we have read enough to fill the entire output buffer, we return - if (cookie->stream.avail_out == 0) - return size; - - if (cookie->done) - return -1; - } -} - -static ssize_t xz_write(void* data, const char* buffer, size_t size) { - struct xz_cookie* cookie = (struct xz_cookie*)data; - if (!cookie) - return -1; - - // Do not write when mode is "r" - if (cookie->mode == 'r') - return -1; - - // Return nothing when there is no input - if (size == 0) - return 0; - - // Set input to allocated buffer - cookie->stream.next_in = (uint8_t *)buffer; - cookie->stream.avail_in = size; - - while (1) { - cookie->stream.next_out = cookie->buffer; - cookie->stream.avail_out = sizeof(cookie->buffer); - - lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN); - if (ret != LZMA_OK) - return -1; - - size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out; - if (bytes_to_write) { - size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f); - - if (bytes_written != bytes_to_write) - return -1; - } - - // Report that all data has been written - if (cookie->stream.avail_in == 0) - return size; - } -} - -static int xz_close(void* data) { - struct xz_cookie* cookie = (struct xz_cookie*)data; - if (!cookie) - return -1; - - if (cookie->mode == 'w') { - while (1) { - cookie->stream.next_out = cookie->buffer; - cookie->stream.avail_out = sizeof(cookie->buffer); - - lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - return -1; - - size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out; - if (bytes_to_write) { - size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f); - - if (bytes_written != bytes_to_write) - return -1; - } - - if (ret == LZMA_STREAM_END) - break; - } - } - - lzma_end(&cookie->stream); - - // Close input file - int r = fclose(cookie->f); - free(cookie); - - return r; -} - -static cookie_io_functions_t xz_functions = { - .read = xz_read, - .write = xz_write, - .seek = NULL, - .close = xz_close, -}; - -FILE* pakfire_xzfopen(FILE* f, const char* mode) { - int level = XZ_COMPRESSION_LEVEL; - lzma_ret ret; - int r; - - if (!f) { - errno = EBADFD; - return NULL; - } - - if (!mode) { - errno = EINVAL; - return NULL; - } - - struct xz_cookie* cookie = calloc(1, sizeof(*cookie)); - if (!cookie) - return NULL; - - cookie->f = f; - cookie->mode = *mode; - - switch (cookie->mode) { - case 'r': - ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0); - break; - - case 'w': - // Try parsing the level - r = parse_level(&level, mode); - if (r < 0) - goto ERROR; - - ret = lzma_easy_encoder(&cookie->stream, level, LZMA_CHECK_SHA256); - break; - - default: - errno = ENOTSUP; - goto ERROR; - } - - if (ret != LZMA_OK) - goto ERROR; - - return fopencookie(cookie, mode, xz_functions); - -ERROR: - free(cookie); - return NULL; -} - -// ZSTD - -struct zstd_cookie { - FILE* f; - char mode; - int done; - - // Encoder - ZSTD_CStream* cstream; - ZSTD_inBuffer in; - - // Decoder - ZSTD_DStream* dstream; - ZSTD_outBuffer out; - - uint8_t buffer[BUFFER_SIZE]; -}; - -static void zstd_free(struct zstd_cookie* cookie) { - if (cookie->cstream) - ZSTD_freeCStream(cookie->cstream); - if (cookie->dstream) - ZSTD_freeDStream(cookie->dstream); - free(cookie); -} - -static ssize_t zstd_read(void* data, char* buffer, size_t size) { - struct zstd_cookie* cookie = data; - int r; - - if (!cookie) - return -1; - - // Do not read when mode is "w" - if (cookie->mode == 'w') - return -1; - - if (cookie->done) - return 0; - - // Configure output buffer - cookie->out.dst = buffer; - cookie->out.size = size; - cookie->out.pos = 0; - - while (cookie->out.pos < cookie->out.size) { - if (cookie->in.pos >= cookie->in.size) { - cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f); - - // EOF? - if (!cookie->in.size) { - if (feof(cookie->f)) - break; - - return -1; - } - - cookie->in.pos = 0; - } - - r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in); - if (ZSTD_isError(r)) - return -1; - } - - return cookie->out.pos; -} - -static ssize_t zstd_write(void* data, const char* buffer, size_t size) { - struct zstd_cookie* cookie = data; - int r; - - if (!cookie) - return -1; - - // Do not write when mode is "r" - if (cookie->mode == 'r') - return -1; - - // Configure input buffer - cookie->in.src = buffer; - cookie->in.size = size; - cookie->in.pos = 0; - - while (cookie->in.pos < cookie->in.size) { - cookie->out.dst = cookie->buffer; - cookie->out.size = sizeof(cookie->buffer); - cookie->out.pos = 0; - - r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in); - if (ZSTD_isError(r)) - return -1; - - r = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f); - if (r < (ssize_t)cookie->out.pos) - return -1; - } - - return size; -} - -static int zstd_flush(void* data) { - struct zstd_cookie* cookie = data; - ssize_t bytes_left; - int r; - - // Fail if were given no cookie - if (!cookie) - return -EINVAL; - - // Reset the input buffer - cookie->in.src = NULL; - cookie->in.size = 0; - cookie->in.pos = 0; - - // In write mode, we will have to flush all buffers - if (cookie->mode == 'w') { - for (;;) { - // Reset output buffer - cookie->out.dst = cookie->buffer; - cookie->out.size = sizeof(cookie->buffer); - cookie->out.pos = 0; - - bytes_left = ZSTD_endStream(cookie->cstream, &cookie->out); - if (ZSTD_isError(bytes_left)) - return -1; - - // Otherwise we write the buffer to the file - if (cookie->out.pos) { - r = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f); - if (r < (ssize_t)cookie->out.pos) - return -1; - } - - // If the buffer is empty we are done - if (bytes_left == 0) - break; - } - } - - return 0; -} - -static int zstd_close(void* data) { - struct zstd_cookie* cookie = data; - int r; - - // Fail if were given no cookie - if (!cookie) - return -EINVAL; - - // In write mode, we will have to flush all buffers - if (cookie->mode == 'w') - zstd_flush(cookie); - - // Close the underlying file handle - r = fclose(cookie->f); - - // Free everything - zstd_free(cookie); - - return r; -} - -static cookie_io_functions_t zstd_functions = { - .read = zstd_read, - .write = zstd_write, - .seek = NULL, - .close = zstd_close, -}; - -FILE* pakfire_zstdfopen(FILE* f, const char* mode) { - struct zstd_cookie* cookie = NULL; - int level = ZSTD_COMPRESSION_LEVEL; - int r; - - // Check for a valid file handle - if (!f) { - errno = EBADFD; - return NULL; - } - - // Check for some mode - if (!mode) { - errno = EINVAL; - return NULL; - } - - // Allocate a new cookie - cookie = calloc(1, sizeof(*cookie)); - if (!cookie) - goto ERROR; - - // Store the file handle - cookie->f = f; - - // Store the mode - cookie->mode = *mode; - - switch (cookie->mode) { - case 'r': - // Allocate stream - cookie->dstream = ZSTD_createDStream(); - if (!cookie->dstream) - goto ERROR; - - // Initialize stream - r = ZSTD_initDStream(cookie->dstream); - if (ZSTD_isError(r)) - goto ERROR; - - cookie->in.src = cookie->buffer; - cookie->in.pos = 0; - cookie->in.size = 0; - break; - - case 'w': - // Try parsing the level - r = parse_level(&level, mode); - if (r < 0) - goto ERROR; - - // Allocate stream - cookie->cstream = ZSTD_createCStream(); - if (!cookie->cstream) - goto ERROR; - - // Initialize stream - r = ZSTD_initCStream(cookie->cstream, level); - if (ZSTD_isError(r)) - goto ERROR; - - cookie->out.dst = cookie->buffer; - cookie->out.pos = 0; - cookie->out.size = sizeof(cookie->buffer); - break; - - default: - errno = ENOTSUP; - goto ERROR; - } - - return fopencookie(cookie, mode, zstd_functions); - -ERROR: - if (cookie) - zstd_free(cookie); - - return NULL; -} - // Common compression struct pakfire_compress { @@ -696,7 +68,7 @@ static int pakfire_copy_data_from_file(struct pakfire_ctx* ctx, struct archive* archive, FILE* f) { ssize_t bytes_written = 0; ssize_t bytes_read = 0; - char buffer[BUFFER_SIZE]; + char buffer[64 * 1024]; // Loop through the entire length of the file while (!feof(f)) { diff --git a/src/pakfire/compress.h b/src/pakfire/compress.h index 60a0839c..2d1797b7 100644 --- a/src/pakfire/compress.h +++ b/src/pakfire/compress.h @@ -28,18 +28,6 @@ #include #include -// Automatically detect -FILE* pakfire_xfopen(FILE* f, const char* mode); - -// gzip -FILE* pakfire_gzfopen(FILE* f, const char* mode); - -// XZ -FILE* pakfire_xzfopen(FILE* f, const char* mode); - -// ZSTD -FILE* pakfire_zstdfopen(FILE* f, const char* mode); - // Algorithms enum pakfire_compressions { PAKFIRE_COMPRESS_ZSTD, diff --git a/src/pakfire/log_file.c b/src/pakfire/log_file.c index 0c961405..7b54c435 100644 --- a/src/pakfire/log_file.c +++ b/src/pakfire/log_file.c @@ -21,11 +21,11 @@ #include #include -#include #include #include #include #include +#include struct pakfire_log_file { struct pakfire_ctx* ctx; diff --git a/src/pakfire/repo.c b/src/pakfire/repo.c index b172b91f..27eed4e7 100644 --- a/src/pakfire/repo.c +++ b/src/pakfire/repo.c @@ -34,7 +34,6 @@ #include #include -#include #include #include #include @@ -50,6 +49,7 @@ #include #include #include +#include #define METADATA_CHECKSUMS (PAKFIRE_HASH_SHA3_512|PAKFIRE_HASH_BLAKE2B512) diff --git a/src/pakfire/xfopen.c b/src/pakfire/xfopen.c new file mode 100644 index 00000000..c32d19c0 --- /dev/null +++ b/src/pakfire/xfopen.c @@ -0,0 +1,660 @@ +/*############################################################################# +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2025 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +#############################################################################*/ + +#include +#include +#include +#include + +// libxz +#include + +// zstd +#include + +// libz +#include + +#include + +// Read up to N bytes for analyze the magic +#define MAX_MAGIC_LENGTH 6 + +// Compression/Decompression buffer size +#define BUFFER_SIZE 64 * 1024 + +// Settings for XZ +#define XZ_COMPRESSION_LEVEL 6 + +// Settings for ZSTD +#define ZSTD_COMPRESSION_LEVEL 15 + +const struct compressor { + char magic[MAX_MAGIC_LENGTH]; + size_t magic_length; + FILE* (*open)(FILE* f, const char* mode); +} compressors[] = { + // Gzip + { { 0x1f, 0x8b }, 2, pakfire_gzfopen, }, + // XZ + { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, }, + // ZSTD + { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, }, + // End + { "", 0, NULL, }, +}; + +// Try to guess the compression +FILE* pakfire_xfopen(FILE* f, const char* mode) { + char buffer[MAX_MAGIC_LENGTH]; + + if (!f) { + errno = EBADFD; + return NULL; + } + + if (!mode) { + errno = EINVAL; + return NULL; + } + + // This only works for reading files + if (*mode != 'r') { + errno = ENOTSUP; + return NULL; + } + + fpos_t pos; + + // Store the position + int r = fgetpos(f, &pos); + if (r < 0) + return NULL; + + // Read a couple of bytes + size_t bytes_read = fread(buffer, 1, sizeof(buffer), f); + + // Reset position + r = fsetpos(f, &pos); + if (r < 0) + return NULL; + + // Check if we could read anything + if (!bytes_read || bytes_read < sizeof(buffer)) + return f; + + // Analyze magic + for (const struct compressor* c = compressors; c->open; c++) { + // Check if we have read enough data + if (bytes_read < c->magic_length) + continue; + + // Compare the magic value + r = memcmp(c->magic, buffer, c->magic_length); + if (r) + continue; + + // We found a match! + return c->open(f, mode); + } + + // Nothing seems to match + errno = ENOTSUP; + return f; +} + +static int parse_level(int* level, const char* mode) { + unsigned long value = 0; + char* p = NULL; + + // We must have mode + if (!mode || *mode != 'w') { + errno = EINVAL; + return -1; + } + + // Skip the w + mode++; + + // Do nothing if we have reached the end of the string + if (!*mode) + return 0; + + // Try parsing a number + value = strtoul(mode, &p, 10); + + // Break on error + if (value == ULONG_MAX) + return -1; + + // Check if we have parsed the entire string + if (p && *p) { + errno = EINVAL; + return -1; + } + + // Otherwise we return the value and return + *level = value; + return 0; +} + +/* + gzip +*/ + +static ssize_t gz_read(void* cookie, char* buffer, size_t size) { + return gzread((gzFile)cookie, buffer, size); +} + +static ssize_t gz_write(void* cookie, const char* buffer, size_t size) { + return gzwrite((gzFile)cookie, buffer, size); +} + +static int gz_close(void* cookie) { + return gzclose((gzFile)cookie); +} + +FILE* pakfire_gzfopen(FILE* f, const char* mode) { + gzFile cookie = NULL; + int fd = -EBADF; + + // Check for a valid file descriptor + if (!f) { + errno = EBADF; + return NULL; + } + + // Fetch the file descriptor + fd = fileno(f); + if (fd < 0) { + errno = EBADF; + return NULL; + } + + // Duplicate the file descriptor to pass it to gzip + fd = dup(fd); + if (fd < 0) + return NULL; + + // Close the original file handle + fclose(f); + + // Open the stream + cookie = gzdopen(fd, mode); + if (!cookie) + return NULL; + + // Increase the buffer size for faster reading + gzbuffer(cookie, 128 * 1024); + + static cookie_io_functions_t gz_functions = { + .read = gz_read, + .write = gz_write, + .close = gz_close, + }; + + return fopencookie(cookie, mode, gz_functions); +} + +struct xz_cookie { + FILE* f; + char mode; + lzma_stream stream; + int done; + + uint8_t buffer[BUFFER_SIZE]; +}; + +static ssize_t xz_read(void* data, char* buffer, size_t size) { + struct xz_cookie* cookie = (struct xz_cookie*)data; + if (!cookie) + return -1; + + // Do not read when mode is "w" + if (cookie->mode == 'w') + return -1; + + lzma_action action = LZMA_RUN; + + // Set output to allocated buffer + cookie->stream.next_out = (uint8_t *)buffer; + cookie->stream.avail_out = size; + + while (1) { + // Read something when the input buffer is empty + if (cookie->stream.avail_in == 0) { + cookie->stream.next_in = cookie->buffer; + cookie->stream.avail_in = fread(cookie->buffer, + 1, sizeof(cookie->buffer), cookie->f); + + // Break if the input file could not be read + if (ferror(cookie->f)) + return -1; + + // Finish after we have reached the end of the input file + if (feof(cookie->f)) + cookie->done = 1; + } + + lzma_ret ret = lzma_code(&cookie->stream, action); + + // If the stream has ended, we just send the + // remaining output and mark that we are done. + if (ret == LZMA_STREAM_END) { + cookie->done = 1; + return size - cookie->stream.avail_out; + } + + // Break on all other unexpected errors + if (ret != LZMA_OK) + return -1; + + // When we have read enough to fill the entire output buffer, we return + if (cookie->stream.avail_out == 0) + return size; + + if (cookie->done) + return -1; + } +} + +static ssize_t xz_write(void* data, const char* buffer, size_t size) { + struct xz_cookie* cookie = (struct xz_cookie*)data; + if (!cookie) + return -1; + + // Do not write when mode is "r" + if (cookie->mode == 'r') + return -1; + + // Return nothing when there is no input + if (size == 0) + return 0; + + // Set input to allocated buffer + cookie->stream.next_in = (uint8_t *)buffer; + cookie->stream.avail_in = size; + + while (1) { + cookie->stream.next_out = cookie->buffer; + cookie->stream.avail_out = sizeof(cookie->buffer); + + lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN); + if (ret != LZMA_OK) + return -1; + + size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out; + if (bytes_to_write) { + size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f); + + if (bytes_written != bytes_to_write) + return -1; + } + + // Report that all data has been written + if (cookie->stream.avail_in == 0) + return size; + } +} + +static int xz_close(void* data) { + struct xz_cookie* cookie = (struct xz_cookie*)data; + if (!cookie) + return -1; + + if (cookie->mode == 'w') { + while (1) { + cookie->stream.next_out = cookie->buffer; + cookie->stream.avail_out = sizeof(cookie->buffer); + + lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + return -1; + + size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out; + if (bytes_to_write) { + size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f); + + if (bytes_written != bytes_to_write) + return -1; + } + + if (ret == LZMA_STREAM_END) + break; + } + } + + lzma_end(&cookie->stream); + + // Close input file + int r = fclose(cookie->f); + free(cookie); + + return r; +} + +static cookie_io_functions_t xz_functions = { + .read = xz_read, + .write = xz_write, + .seek = NULL, + .close = xz_close, +}; + +FILE* pakfire_xzfopen(FILE* f, const char* mode) { + int level = XZ_COMPRESSION_LEVEL; + lzma_ret ret; + int r; + + if (!f) { + errno = EBADFD; + return NULL; + } + + if (!mode) { + errno = EINVAL; + return NULL; + } + + struct xz_cookie* cookie = calloc(1, sizeof(*cookie)); + if (!cookie) + return NULL; + + cookie->f = f; + cookie->mode = *mode; + + switch (cookie->mode) { + case 'r': + ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0); + break; + + case 'w': + // Try parsing the level + r = parse_level(&level, mode); + if (r < 0) + goto ERROR; + + ret = lzma_easy_encoder(&cookie->stream, level, LZMA_CHECK_SHA256); + break; + + default: + errno = ENOTSUP; + goto ERROR; + } + + if (ret != LZMA_OK) + goto ERROR; + + return fopencookie(cookie, mode, xz_functions); + +ERROR: + free(cookie); + return NULL; +} + +// ZSTD + +struct zstd_cookie { + FILE* f; + char mode; + int done; + + // Encoder + ZSTD_CStream* cstream; + ZSTD_inBuffer in; + + // Decoder + ZSTD_DStream* dstream; + ZSTD_outBuffer out; + + uint8_t buffer[BUFFER_SIZE]; +}; + +static void zstd_free(struct zstd_cookie* cookie) { + if (cookie->cstream) + ZSTD_freeCStream(cookie->cstream); + if (cookie->dstream) + ZSTD_freeDStream(cookie->dstream); + free(cookie); +} + +static ssize_t zstd_read(void* data, char* buffer, size_t size) { + struct zstd_cookie* cookie = data; + int r; + + if (!cookie) + return -1; + + // Do not read when mode is "w" + if (cookie->mode == 'w') + return -1; + + if (cookie->done) + return 0; + + // Configure output buffer + cookie->out.dst = buffer; + cookie->out.size = size; + cookie->out.pos = 0; + + while (cookie->out.pos < cookie->out.size) { + if (cookie->in.pos >= cookie->in.size) { + cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f); + + // EOF? + if (!cookie->in.size) { + if (feof(cookie->f)) + break; + + return -1; + } + + cookie->in.pos = 0; + } + + r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in); + if (ZSTD_isError(r)) + return -1; + } + + return cookie->out.pos; +} + +static ssize_t zstd_write(void* data, const char* buffer, size_t size) { + struct zstd_cookie* cookie = data; + int r; + + if (!cookie) + return -1; + + // Do not write when mode is "r" + if (cookie->mode == 'r') + return -1; + + // Configure input buffer + cookie->in.src = buffer; + cookie->in.size = size; + cookie->in.pos = 0; + + while (cookie->in.pos < cookie->in.size) { + cookie->out.dst = cookie->buffer; + cookie->out.size = sizeof(cookie->buffer); + cookie->out.pos = 0; + + r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in); + if (ZSTD_isError(r)) + return -1; + + r = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f); + if (r < (ssize_t)cookie->out.pos) + return -1; + } + + return size; +} + +static int zstd_flush(void* data) { + struct zstd_cookie* cookie = data; + ssize_t bytes_left; + int r; + + // Fail if were given no cookie + if (!cookie) + return -EINVAL; + + // Reset the input buffer + cookie->in.src = NULL; + cookie->in.size = 0; + cookie->in.pos = 0; + + // In write mode, we will have to flush all buffers + if (cookie->mode == 'w') { + for (;;) { + // Reset output buffer + cookie->out.dst = cookie->buffer; + cookie->out.size = sizeof(cookie->buffer); + cookie->out.pos = 0; + + bytes_left = ZSTD_endStream(cookie->cstream, &cookie->out); + if (ZSTD_isError(bytes_left)) + return -1; + + // Otherwise we write the buffer to the file + if (cookie->out.pos) { + r = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f); + if (r < (ssize_t)cookie->out.pos) + return -1; + } + + // If the buffer is empty we are done + if (bytes_left == 0) + break; + } + } + + return 0; +} + +static int zstd_close(void* data) { + struct zstd_cookie* cookie = data; + int r; + + // Fail if were given no cookie + if (!cookie) + return -EINVAL; + + // In write mode, we will have to flush all buffers + if (cookie->mode == 'w') + zstd_flush(cookie); + + // Close the underlying file handle + r = fclose(cookie->f); + + // Free everything + zstd_free(cookie); + + return r; +} + +static cookie_io_functions_t zstd_functions = { + .read = zstd_read, + .write = zstd_write, + .seek = NULL, + .close = zstd_close, +}; + +FILE* pakfire_zstdfopen(FILE* f, const char* mode) { + struct zstd_cookie* cookie = NULL; + int level = ZSTD_COMPRESSION_LEVEL; + int r; + + // Check for a valid file handle + if (!f) { + errno = EBADFD; + return NULL; + } + + // Check for some mode + if (!mode) { + errno = EINVAL; + return NULL; + } + + // Allocate a new cookie + cookie = calloc(1, sizeof(*cookie)); + if (!cookie) + goto ERROR; + + // Store the file handle + cookie->f = f; + + // Store the mode + cookie->mode = *mode; + + switch (cookie->mode) { + case 'r': + // Allocate stream + cookie->dstream = ZSTD_createDStream(); + if (!cookie->dstream) + goto ERROR; + + // Initialize stream + r = ZSTD_initDStream(cookie->dstream); + if (ZSTD_isError(r)) + goto ERROR; + + cookie->in.src = cookie->buffer; + cookie->in.pos = 0; + cookie->in.size = 0; + break; + + case 'w': + // Try parsing the level + r = parse_level(&level, mode); + if (r < 0) + goto ERROR; + + // Allocate stream + cookie->cstream = ZSTD_createCStream(); + if (!cookie->cstream) + goto ERROR; + + // Initialize stream + r = ZSTD_initCStream(cookie->cstream, level); + if (ZSTD_isError(r)) + goto ERROR; + + cookie->out.dst = cookie->buffer; + cookie->out.pos = 0; + cookie->out.size = sizeof(cookie->buffer); + break; + + default: + errno = ENOTSUP; + goto ERROR; + } + + return fopencookie(cookie, mode, zstd_functions); + +ERROR: + if (cookie) + zstd_free(cookie); + + return NULL; +} diff --git a/src/pakfire/xfopen.h b/src/pakfire/xfopen.h new file mode 100644 index 00000000..526633c6 --- /dev/null +++ b/src/pakfire/xfopen.h @@ -0,0 +1,38 @@ +/*############################################################################# +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2025 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +#############################################################################*/ + +#ifndef PAKFIRE_XFOPEN_H +#define PAKFIRE_XFOPEN_H + +#include + +// Automatically detect +FILE* pakfire_xfopen(FILE* f, const char* mode); + +// gzip +FILE* pakfire_gzfopen(FILE* f, const char* mode); + +// XZ +FILE* pakfire_xzfopen(FILE* f, const char* mode); + +// ZSTD +FILE* pakfire_zstdfopen(FILE* f, const char* mode); + +#endif /* PAKFIRE_XFOPEN_H */ diff --git a/tests/libpakfire/compress.c b/tests/libpakfire/xfopen.c similarity index 99% rename from tests/libpakfire/compress.c rename to tests/libpakfire/xfopen.c index abeca34e..5f7a417e 100644 --- a/tests/libpakfire/compress.c +++ b/tests/libpakfire/xfopen.c @@ -22,9 +22,9 @@ #include #include -#include #include #include +#include #include "../testsuite.h"