/tests/libpakfire/archive
/tests/libpakfire/build
/tests/libpakfire/cgroup
-/tests/libpakfire/compress
/tests/libpakfire/config
/tests/libpakfire/daemon
/tests/libpakfire/db
/tests/libpakfire/string
/tests/libpakfire/util
/tests/libpakfire/xfer
+/tests/libpakfire/xfopen
/tests/parser/test
/tests/stub/root
/tmp
src/pakfire/util.c \
src/pakfire/util.h \
src/pakfire/xfer.c \
- src/pakfire/xfer.h
+ src/pakfire/xfer.h \
+ src/pakfire/xfopen.c \
+ src/pakfire/xfopen.h
libpakfire_la_CFLAGS = \
$(AM_CFLAGS) \
tests/libpakfire/archive \
tests/libpakfire/build \
tests/libpakfire/cgroup \
- tests/libpakfire/compress \
tests/libpakfire/config \
tests/libpakfire/daemon \
tests/libpakfire/db \
tests/libpakfire/repo \
tests/libpakfire/string \
tests/libpakfire/util \
- tests/libpakfire/xfer
+ tests/libpakfire/xfer \
+ tests/libpakfire/xfopen
dist_tests_libpakfire_main_SOURCES = \
tests/libpakfire/main.c
tests_libpakfire_cgroup_LDADD = \
$(TESTSUITE_LDADD)
-dist_tests_libpakfire_compress_SOURCES = \
- tests/libpakfire/compress.c
-
-tests_libpakfire_compress_CPPFLAGS = \
- $(TESTSUITE_CPPFLAGS)
-
-tests_libpakfire_compress_CFLAGS = \
- $(TESTSUITE_CFLAGS)
-
-tests_libpakfire_compress_LDFLAGS = \
- $(TESTSUITE_LDFLAGS)
-
-tests_libpakfire_compress_LDADD = \
- $(TESTSUITE_LDADD)
-
dist_tests_libpakfire_config_SOURCES = \
tests/libpakfire/config.c
tests_libpakfire_xfer_LDADD = \
$(TESTSUITE_LDADD)
+dist_tests_libpakfire_xfopen_SOURCES = \
+ tests/libpakfire/xfopen.c
+
+tests_libpakfire_xfopen_CPPFLAGS = \
+ $(TESTSUITE_CPPFLAGS)
+
+tests_libpakfire_xfopen_CFLAGS = \
+ $(TESTSUITE_CFLAGS)
+
+tests_libpakfire_xfopen_LDFLAGS = \
+ $(TESTSUITE_LDFLAGS)
+
+tests_libpakfire_xfopen_LDADD = \
+ $(TESTSUITE_LDADD)
+
# ------------------------------------------------------------------------------
noinst_PROGRAMS += \
#include <string.h>
#include <archive.h>
-#include <lzma.h>
-#include <zstd.h>
-#include <zlib.h>
#include <pakfire/ctx.h>
#include <pakfire/compress.h>
#include <pakfire/string.h>
#include <pakfire/util.h>
-// Read up to N bytes for analyze the magic
-#define MAX_MAGIC_LENGTH 6
-
-// Compression/Decompression buffer size
-#define BUFFER_SIZE 64 * 1024
-
-// Settings for XZ
-#define XZ_COMPRESSION_LEVEL 6
-
-// Settings for ZSTD
-#define ZSTD_COMPRESSION_LEVEL 15
-
-const struct compressor {
- char magic[MAX_MAGIC_LENGTH];
- size_t magic_length;
- FILE* (*open)(FILE* f, const char* mode);
-} compressors[] = {
- // Gzip
- { { 0x1f, 0x8b }, 2, pakfire_gzfopen, },
- // XZ
- { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
- // ZSTD
- { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
- // End
- { "", 0, NULL, },
-};
-
-// Try to guess the compression
-FILE* pakfire_xfopen(FILE* f, const char* mode) {
- char buffer[MAX_MAGIC_LENGTH];
-
- if (!f) {
- errno = EBADFD;
- return NULL;
- }
-
- if (!mode) {
- errno = EINVAL;
- return NULL;
- }
-
- // This only works for reading files
- if (*mode != 'r') {
- errno = ENOTSUP;
- return NULL;
- }
-
- fpos_t pos;
-
- // Store the position
- int r = fgetpos(f, &pos);
- if (r < 0)
- return NULL;
-
- // Read a couple of bytes
- size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
-
- // Reset position
- r = fsetpos(f, &pos);
- if (r < 0)
- return NULL;
-
- // Check if we could read anything
- if (!bytes_read || bytes_read < sizeof(buffer))
- return f;
-
- // Analyze magic
- for (const struct compressor* c = compressors; c->open; c++) {
- // Check if we have read enough data
- if (bytes_read < c->magic_length)
- continue;
-
- // Compare the magic value
- r = memcmp(c->magic, buffer, c->magic_length);
- if (r)
- continue;
-
- // We found a match!
- return c->open(f, mode);
- }
-
- // Nothing seems to match
- errno = ENOTSUP;
- return f;
-}
-
-static int parse_level(int* level, const char* mode) {
- unsigned long value = 0;
- char* p = NULL;
-
- // We must have mode
- if (!mode || *mode != 'w') {
- errno = EINVAL;
- return -1;
- }
-
- // Skip the w
- mode++;
-
- // Do nothing if we have reached the end of the string
- if (!*mode)
- return 0;
-
- // Try parsing a number
- value = strtoul(mode, &p, 10);
-
- // Break on error
- if (value == ULONG_MAX)
- return -1;
-
- // Check if we have parsed the entire string
- if (p && *p) {
- errno = EINVAL;
- return -1;
- }
-
- // Otherwise we return the value and return
- *level = value;
- return 0;
-}
-
-/*
- gzip
-*/
-
-static ssize_t gz_read(void* cookie, char* buffer, size_t size) {
- return gzread((gzFile)cookie, buffer, size);
-}
-
-static ssize_t gz_write(void* cookie, const char* buffer, size_t size) {
- return gzwrite((gzFile)cookie, buffer, size);
-}
-
-static int gz_close(void* cookie) {
- return gzclose((gzFile)cookie);
-}
-
-FILE* pakfire_gzfopen(FILE* f, const char* mode) {
- gzFile cookie = NULL;
- int fd = -EBADF;
-
- // Check for a valid file descriptor
- if (!f) {
- errno = EBADF;
- return NULL;
- }
-
- // Fetch the file descriptor
- fd = fileno(f);
- if (fd < 0) {
- errno = EBADF;
- return NULL;
- }
-
- // Duplicate the file descriptor to pass it to gzip
- fd = dup(fd);
- if (fd < 0)
- return NULL;
-
- // Close the original file handle
- fclose(f);
-
- // Open the stream
- cookie = gzdopen(fd, mode);
- if (!cookie)
- return NULL;
-
- // Increase the buffer size for faster reading
- gzbuffer(cookie, 128 * 1024);
-
- static cookie_io_functions_t gz_functions = {
- .read = gz_read,
- .write = gz_write,
- .close = gz_close,
- };
-
- return fopencookie(cookie, mode, gz_functions);
-}
-
-struct xz_cookie {
- FILE* f;
- char mode;
- lzma_stream stream;
- int done;
-
- uint8_t buffer[BUFFER_SIZE];
-};
-
-static ssize_t xz_read(void* data, char* buffer, size_t size) {
- struct xz_cookie* cookie = (struct xz_cookie*)data;
- if (!cookie)
- return -1;
-
- // Do not read when mode is "w"
- if (cookie->mode == 'w')
- return -1;
-
- lzma_action action = LZMA_RUN;
-
- // Set output to allocated buffer
- cookie->stream.next_out = (uint8_t *)buffer;
- cookie->stream.avail_out = size;
-
- while (1) {
- // Read something when the input buffer is empty
- if (cookie->stream.avail_in == 0) {
- cookie->stream.next_in = cookie->buffer;
- cookie->stream.avail_in = fread(cookie->buffer,
- 1, sizeof(cookie->buffer), cookie->f);
-
- // Break if the input file could not be read
- if (ferror(cookie->f))
- return -1;
-
- // Finish after we have reached the end of the input file
- if (feof(cookie->f))
- cookie->done = 1;
- }
-
- lzma_ret ret = lzma_code(&cookie->stream, action);
-
- // If the stream has ended, we just send the
- // remaining output and mark that we are done.
- if (ret == LZMA_STREAM_END) {
- cookie->done = 1;
- return size - cookie->stream.avail_out;
- }
-
- // Break on all other unexpected errors
- if (ret != LZMA_OK)
- return -1;
-
- // When we have read enough to fill the entire output buffer, we return
- if (cookie->stream.avail_out == 0)
- return size;
-
- if (cookie->done)
- return -1;
- }
-}
-
-static ssize_t xz_write(void* data, const char* buffer, size_t size) {
- struct xz_cookie* cookie = (struct xz_cookie*)data;
- if (!cookie)
- return -1;
-
- // Do not write when mode is "r"
- if (cookie->mode == 'r')
- return -1;
-
- // Return nothing when there is no input
- if (size == 0)
- return 0;
-
- // Set input to allocated buffer
- cookie->stream.next_in = (uint8_t *)buffer;
- cookie->stream.avail_in = size;
-
- while (1) {
- cookie->stream.next_out = cookie->buffer;
- cookie->stream.avail_out = sizeof(cookie->buffer);
-
- lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
- if (ret != LZMA_OK)
- return -1;
-
- size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
- if (bytes_to_write) {
- size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
-
- if (bytes_written != bytes_to_write)
- return -1;
- }
-
- // Report that all data has been written
- if (cookie->stream.avail_in == 0)
- return size;
- }
-}
-
-static int xz_close(void* data) {
- struct xz_cookie* cookie = (struct xz_cookie*)data;
- if (!cookie)
- return -1;
-
- if (cookie->mode == 'w') {
- while (1) {
- cookie->stream.next_out = cookie->buffer;
- cookie->stream.avail_out = sizeof(cookie->buffer);
-
- lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
- if (ret != LZMA_OK && ret != LZMA_STREAM_END)
- return -1;
-
- size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
- if (bytes_to_write) {
- size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
-
- if (bytes_written != bytes_to_write)
- return -1;
- }
-
- if (ret == LZMA_STREAM_END)
- break;
- }
- }
-
- lzma_end(&cookie->stream);
-
- // Close input file
- int r = fclose(cookie->f);
- free(cookie);
-
- return r;
-}
-
-static cookie_io_functions_t xz_functions = {
- .read = xz_read,
- .write = xz_write,
- .seek = NULL,
- .close = xz_close,
-};
-
-FILE* pakfire_xzfopen(FILE* f, const char* mode) {
- int level = XZ_COMPRESSION_LEVEL;
- lzma_ret ret;
- int r;
-
- if (!f) {
- errno = EBADFD;
- return NULL;
- }
-
- if (!mode) {
- errno = EINVAL;
- return NULL;
- }
-
- struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
- if (!cookie)
- return NULL;
-
- cookie->f = f;
- cookie->mode = *mode;
-
- switch (cookie->mode) {
- case 'r':
- ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
- break;
-
- case 'w':
- // Try parsing the level
- r = parse_level(&level, mode);
- if (r < 0)
- goto ERROR;
-
- ret = lzma_easy_encoder(&cookie->stream, level, LZMA_CHECK_SHA256);
- break;
-
- default:
- errno = ENOTSUP;
- goto ERROR;
- }
-
- if (ret != LZMA_OK)
- goto ERROR;
-
- return fopencookie(cookie, mode, xz_functions);
-
-ERROR:
- free(cookie);
- return NULL;
-}
-
-// ZSTD
-
-struct zstd_cookie {
- FILE* f;
- char mode;
- int done;
-
- // Encoder
- ZSTD_CStream* cstream;
- ZSTD_inBuffer in;
-
- // Decoder
- ZSTD_DStream* dstream;
- ZSTD_outBuffer out;
-
- uint8_t buffer[BUFFER_SIZE];
-};
-
-static void zstd_free(struct zstd_cookie* cookie) {
- if (cookie->cstream)
- ZSTD_freeCStream(cookie->cstream);
- if (cookie->dstream)
- ZSTD_freeDStream(cookie->dstream);
- free(cookie);
-}
-
-static ssize_t zstd_read(void* data, char* buffer, size_t size) {
- struct zstd_cookie* cookie = data;
- int r;
-
- if (!cookie)
- return -1;
-
- // Do not read when mode is "w"
- if (cookie->mode == 'w')
- return -1;
-
- if (cookie->done)
- return 0;
-
- // Configure output buffer
- cookie->out.dst = buffer;
- cookie->out.size = size;
- cookie->out.pos = 0;
-
- while (cookie->out.pos < cookie->out.size) {
- if (cookie->in.pos >= cookie->in.size) {
- cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
-
- // EOF?
- if (!cookie->in.size) {
- if (feof(cookie->f))
- break;
-
- return -1;
- }
-
- cookie->in.pos = 0;
- }
-
- r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
- if (ZSTD_isError(r))
- return -1;
- }
-
- return cookie->out.pos;
-}
-
-static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
- struct zstd_cookie* cookie = data;
- int r;
-
- if (!cookie)
- return -1;
-
- // Do not write when mode is "r"
- if (cookie->mode == 'r')
- return -1;
-
- // Configure input buffer
- cookie->in.src = buffer;
- cookie->in.size = size;
- cookie->in.pos = 0;
-
- while (cookie->in.pos < cookie->in.size) {
- cookie->out.dst = cookie->buffer;
- cookie->out.size = sizeof(cookie->buffer);
- cookie->out.pos = 0;
-
- r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
- if (ZSTD_isError(r))
- return -1;
-
- r = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
- if (r < (ssize_t)cookie->out.pos)
- return -1;
- }
-
- return size;
-}
-
-static int zstd_flush(void* data) {
- struct zstd_cookie* cookie = data;
- ssize_t bytes_left;
- int r;
-
- // Fail if were given no cookie
- if (!cookie)
- return -EINVAL;
-
- // Reset the input buffer
- cookie->in.src = NULL;
- cookie->in.size = 0;
- cookie->in.pos = 0;
-
- // In write mode, we will have to flush all buffers
- if (cookie->mode == 'w') {
- for (;;) {
- // Reset output buffer
- cookie->out.dst = cookie->buffer;
- cookie->out.size = sizeof(cookie->buffer);
- cookie->out.pos = 0;
-
- bytes_left = ZSTD_endStream(cookie->cstream, &cookie->out);
- if (ZSTD_isError(bytes_left))
- return -1;
-
- // Otherwise we write the buffer to the file
- if (cookie->out.pos) {
- r = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
- if (r < (ssize_t)cookie->out.pos)
- return -1;
- }
-
- // If the buffer is empty we are done
- if (bytes_left == 0)
- break;
- }
- }
-
- return 0;
-}
-
-static int zstd_close(void* data) {
- struct zstd_cookie* cookie = data;
- int r;
-
- // Fail if were given no cookie
- if (!cookie)
- return -EINVAL;
-
- // In write mode, we will have to flush all buffers
- if (cookie->mode == 'w')
- zstd_flush(cookie);
-
- // Close the underlying file handle
- r = fclose(cookie->f);
-
- // Free everything
- zstd_free(cookie);
-
- return r;
-}
-
-static cookie_io_functions_t zstd_functions = {
- .read = zstd_read,
- .write = zstd_write,
- .seek = NULL,
- .close = zstd_close,
-};
-
-FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
- struct zstd_cookie* cookie = NULL;
- int level = ZSTD_COMPRESSION_LEVEL;
- int r;
-
- // Check for a valid file handle
- if (!f) {
- errno = EBADFD;
- return NULL;
- }
-
- // Check for some mode
- if (!mode) {
- errno = EINVAL;
- return NULL;
- }
-
- // Allocate a new cookie
- cookie = calloc(1, sizeof(*cookie));
- if (!cookie)
- goto ERROR;
-
- // Store the file handle
- cookie->f = f;
-
- // Store the mode
- cookie->mode = *mode;
-
- switch (cookie->mode) {
- case 'r':
- // Allocate stream
- cookie->dstream = ZSTD_createDStream();
- if (!cookie->dstream)
- goto ERROR;
-
- // Initialize stream
- r = ZSTD_initDStream(cookie->dstream);
- if (ZSTD_isError(r))
- goto ERROR;
-
- cookie->in.src = cookie->buffer;
- cookie->in.pos = 0;
- cookie->in.size = 0;
- break;
-
- case 'w':
- // Try parsing the level
- r = parse_level(&level, mode);
- if (r < 0)
- goto ERROR;
-
- // Allocate stream
- cookie->cstream = ZSTD_createCStream();
- if (!cookie->cstream)
- goto ERROR;
-
- // Initialize stream
- r = ZSTD_initCStream(cookie->cstream, level);
- if (ZSTD_isError(r))
- goto ERROR;
-
- cookie->out.dst = cookie->buffer;
- cookie->out.pos = 0;
- cookie->out.size = sizeof(cookie->buffer);
- break;
-
- default:
- errno = ENOTSUP;
- goto ERROR;
- }
-
- return fopencookie(cookie, mode, zstd_functions);
-
-ERROR:
- if (cookie)
- zstd_free(cookie);
-
- return NULL;
-}
-
// Common compression
struct pakfire_compress {
struct archive* archive, FILE* f) {
ssize_t bytes_written = 0;
ssize_t bytes_read = 0;
- char buffer[BUFFER_SIZE];
+ char buffer[64 * 1024];
// Loop through the entire length of the file
while (!feof(f)) {
#include <pakfire/hashes.h>
#include <pakfire/pakfire.h>
-// Automatically detect
-FILE* pakfire_xfopen(FILE* f, const char* mode);
-
-// gzip
-FILE* pakfire_gzfopen(FILE* f, const char* mode);
-
-// XZ
-FILE* pakfire_xzfopen(FILE* f, const char* mode);
-
-// ZSTD
-FILE* pakfire_zstdfopen(FILE* f, const char* mode);
-
// Algorithms
enum pakfire_compressions {
PAKFIRE_COMPRESS_ZSTD,
#include <errno.h>
#include <stdlib.h>
-#include <pakfire/compress.h>
#include <pakfire/ctx.h>
#include <pakfire/log_file.h>
#include <pakfire/string.h>
#include <pakfire/util.h>
+#include <pakfire/xfopen.h>
struct pakfire_log_file {
struct pakfire_ctx* ctx;
#include <json.h>
#include <pakfire/archive.h>
-#include <pakfire/compress.h>
#include <pakfire/config.h>
#include <pakfire/constants.h>
#include <pakfire/ctx.h>
#include <pakfire/string.h>
#include <pakfire/util.h>
#include <pakfire/xfer.h>
+#include <pakfire/xfopen.h>
#define METADATA_CHECKSUMS (PAKFIRE_HASH_SHA3_512|PAKFIRE_HASH_BLAKE2B512)
--- /dev/null
+/*#############################################################################
+# #
+# Pakfire - The IPFire package management system #
+# Copyright (C) 2025 Pakfire development team #
+# #
+# This program is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# This program is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with this program. If not, see <http://www.gnu.org/licenses/>. #
+# #
+#############################################################################*/
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// libxz
+#include <lzma.h>
+
+// zstd
+#include <zstd.h>
+
+// libz
+#include <zlib.h>
+
+#include <pakfire/xfopen.h>
+
+// Read up to N bytes for analyze the magic
+#define MAX_MAGIC_LENGTH 6
+
+// Compression/Decompression buffer size
+#define BUFFER_SIZE 64 * 1024
+
+// Settings for XZ
+#define XZ_COMPRESSION_LEVEL 6
+
+// Settings for ZSTD
+#define ZSTD_COMPRESSION_LEVEL 15
+
+const struct compressor {
+ char magic[MAX_MAGIC_LENGTH];
+ size_t magic_length;
+ FILE* (*open)(FILE* f, const char* mode);
+} compressors[] = {
+ // Gzip
+ { { 0x1f, 0x8b }, 2, pakfire_gzfopen, },
+ // XZ
+ { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
+ // ZSTD
+ { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
+ // End
+ { "", 0, NULL, },
+};
+
+// Try to guess the compression
+FILE* pakfire_xfopen(FILE* f, const char* mode) {
+ char buffer[MAX_MAGIC_LENGTH];
+
+ if (!f) {
+ errno = EBADFD;
+ return NULL;
+ }
+
+ if (!mode) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ // This only works for reading files
+ if (*mode != 'r') {
+ errno = ENOTSUP;
+ return NULL;
+ }
+
+ fpos_t pos;
+
+ // Store the position
+ int r = fgetpos(f, &pos);
+ if (r < 0)
+ return NULL;
+
+ // Read a couple of bytes
+ size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
+
+ // Reset position
+ r = fsetpos(f, &pos);
+ if (r < 0)
+ return NULL;
+
+ // Check if we could read anything
+ if (!bytes_read || bytes_read < sizeof(buffer))
+ return f;
+
+ // Analyze magic
+ for (const struct compressor* c = compressors; c->open; c++) {
+ // Check if we have read enough data
+ if (bytes_read < c->magic_length)
+ continue;
+
+ // Compare the magic value
+ r = memcmp(c->magic, buffer, c->magic_length);
+ if (r)
+ continue;
+
+ // We found a match!
+ return c->open(f, mode);
+ }
+
+ // Nothing seems to match
+ errno = ENOTSUP;
+ return f;
+}
+
+static int parse_level(int* level, const char* mode) {
+ unsigned long value = 0;
+ char* p = NULL;
+
+ // We must have mode
+ if (!mode || *mode != 'w') {
+ errno = EINVAL;
+ return -1;
+ }
+
+ // Skip the w
+ mode++;
+
+ // Do nothing if we have reached the end of the string
+ if (!*mode)
+ return 0;
+
+ // Try parsing a number
+ value = strtoul(mode, &p, 10);
+
+ // Break on error
+ if (value == ULONG_MAX)
+ return -1;
+
+ // Check if we have parsed the entire string
+ if (p && *p) {
+ errno = EINVAL;
+ return -1;
+ }
+
+ // Otherwise we return the value and return
+ *level = value;
+ return 0;
+}
+
+/*
+ gzip
+*/
+
+static ssize_t gz_read(void* cookie, char* buffer, size_t size) {
+ return gzread((gzFile)cookie, buffer, size);
+}
+
+static ssize_t gz_write(void* cookie, const char* buffer, size_t size) {
+ return gzwrite((gzFile)cookie, buffer, size);
+}
+
+static int gz_close(void* cookie) {
+ return gzclose((gzFile)cookie);
+}
+
+FILE* pakfire_gzfopen(FILE* f, const char* mode) {
+ gzFile cookie = NULL;
+ int fd = -EBADF;
+
+ // Check for a valid file descriptor
+ if (!f) {
+ errno = EBADF;
+ return NULL;
+ }
+
+ // Fetch the file descriptor
+ fd = fileno(f);
+ if (fd < 0) {
+ errno = EBADF;
+ return NULL;
+ }
+
+ // Duplicate the file descriptor to pass it to gzip
+ fd = dup(fd);
+ if (fd < 0)
+ return NULL;
+
+ // Close the original file handle
+ fclose(f);
+
+ // Open the stream
+ cookie = gzdopen(fd, mode);
+ if (!cookie)
+ return NULL;
+
+ // Increase the buffer size for faster reading
+ gzbuffer(cookie, 128 * 1024);
+
+ static cookie_io_functions_t gz_functions = {
+ .read = gz_read,
+ .write = gz_write,
+ .close = gz_close,
+ };
+
+ return fopencookie(cookie, mode, gz_functions);
+}
+
+struct xz_cookie {
+ FILE* f;
+ char mode;
+ lzma_stream stream;
+ int done;
+
+ uint8_t buffer[BUFFER_SIZE];
+};
+
+static ssize_t xz_read(void* data, char* buffer, size_t size) {
+ struct xz_cookie* cookie = (struct xz_cookie*)data;
+ if (!cookie)
+ return -1;
+
+ // Do not read when mode is "w"
+ if (cookie->mode == 'w')
+ return -1;
+
+ lzma_action action = LZMA_RUN;
+
+ // Set output to allocated buffer
+ cookie->stream.next_out = (uint8_t *)buffer;
+ cookie->stream.avail_out = size;
+
+ while (1) {
+ // Read something when the input buffer is empty
+ if (cookie->stream.avail_in == 0) {
+ cookie->stream.next_in = cookie->buffer;
+ cookie->stream.avail_in = fread(cookie->buffer,
+ 1, sizeof(cookie->buffer), cookie->f);
+
+ // Break if the input file could not be read
+ if (ferror(cookie->f))
+ return -1;
+
+ // Finish after we have reached the end of the input file
+ if (feof(cookie->f))
+ cookie->done = 1;
+ }
+
+ lzma_ret ret = lzma_code(&cookie->stream, action);
+
+ // If the stream has ended, we just send the
+ // remaining output and mark that we are done.
+ if (ret == LZMA_STREAM_END) {
+ cookie->done = 1;
+ return size - cookie->stream.avail_out;
+ }
+
+ // Break on all other unexpected errors
+ if (ret != LZMA_OK)
+ return -1;
+
+ // When we have read enough to fill the entire output buffer, we return
+ if (cookie->stream.avail_out == 0)
+ return size;
+
+ if (cookie->done)
+ return -1;
+ }
+}
+
+static ssize_t xz_write(void* data, const char* buffer, size_t size) {
+ struct xz_cookie* cookie = (struct xz_cookie*)data;
+ if (!cookie)
+ return -1;
+
+ // Do not write when mode is "r"
+ if (cookie->mode == 'r')
+ return -1;
+
+ // Return nothing when there is no input
+ if (size == 0)
+ return 0;
+
+ // Set input to allocated buffer
+ cookie->stream.next_in = (uint8_t *)buffer;
+ cookie->stream.avail_in = size;
+
+ while (1) {
+ cookie->stream.next_out = cookie->buffer;
+ cookie->stream.avail_out = sizeof(cookie->buffer);
+
+ lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
+ if (ret != LZMA_OK)
+ return -1;
+
+ size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
+ if (bytes_to_write) {
+ size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
+
+ if (bytes_written != bytes_to_write)
+ return -1;
+ }
+
+ // Report that all data has been written
+ if (cookie->stream.avail_in == 0)
+ return size;
+ }
+}
+
+static int xz_close(void* data) {
+ struct xz_cookie* cookie = (struct xz_cookie*)data;
+ if (!cookie)
+ return -1;
+
+ if (cookie->mode == 'w') {
+ while (1) {
+ cookie->stream.next_out = cookie->buffer;
+ cookie->stream.avail_out = sizeof(cookie->buffer);
+
+ lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
+ if (ret != LZMA_OK && ret != LZMA_STREAM_END)
+ return -1;
+
+ size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
+ if (bytes_to_write) {
+ size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
+
+ if (bytes_written != bytes_to_write)
+ return -1;
+ }
+
+ if (ret == LZMA_STREAM_END)
+ break;
+ }
+ }
+
+ lzma_end(&cookie->stream);
+
+ // Close input file
+ int r = fclose(cookie->f);
+ free(cookie);
+
+ return r;
+}
+
+static cookie_io_functions_t xz_functions = {
+ .read = xz_read,
+ .write = xz_write,
+ .seek = NULL,
+ .close = xz_close,
+};
+
+FILE* pakfire_xzfopen(FILE* f, const char* mode) {
+ int level = XZ_COMPRESSION_LEVEL;
+ lzma_ret ret;
+ int r;
+
+ if (!f) {
+ errno = EBADFD;
+ return NULL;
+ }
+
+ if (!mode) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
+ if (!cookie)
+ return NULL;
+
+ cookie->f = f;
+ cookie->mode = *mode;
+
+ switch (cookie->mode) {
+ case 'r':
+ ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
+ break;
+
+ case 'w':
+ // Try parsing the level
+ r = parse_level(&level, mode);
+ if (r < 0)
+ goto ERROR;
+
+ ret = lzma_easy_encoder(&cookie->stream, level, LZMA_CHECK_SHA256);
+ break;
+
+ default:
+ errno = ENOTSUP;
+ goto ERROR;
+ }
+
+ if (ret != LZMA_OK)
+ goto ERROR;
+
+ return fopencookie(cookie, mode, xz_functions);
+
+ERROR:
+ free(cookie);
+ return NULL;
+}
+
+// ZSTD
+
+struct zstd_cookie {
+ FILE* f;
+ char mode;
+ int done;
+
+ // Encoder
+ ZSTD_CStream* cstream;
+ ZSTD_inBuffer in;
+
+ // Decoder
+ ZSTD_DStream* dstream;
+ ZSTD_outBuffer out;
+
+ uint8_t buffer[BUFFER_SIZE];
+};
+
+static void zstd_free(struct zstd_cookie* cookie) {
+ if (cookie->cstream)
+ ZSTD_freeCStream(cookie->cstream);
+ if (cookie->dstream)
+ ZSTD_freeDStream(cookie->dstream);
+ free(cookie);
+}
+
+static ssize_t zstd_read(void* data, char* buffer, size_t size) {
+ struct zstd_cookie* cookie = data;
+ int r;
+
+ if (!cookie)
+ return -1;
+
+ // Do not read when mode is "w"
+ if (cookie->mode == 'w')
+ return -1;
+
+ if (cookie->done)
+ return 0;
+
+ // Configure output buffer
+ cookie->out.dst = buffer;
+ cookie->out.size = size;
+ cookie->out.pos = 0;
+
+ while (cookie->out.pos < cookie->out.size) {
+ if (cookie->in.pos >= cookie->in.size) {
+ cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
+
+ // EOF?
+ if (!cookie->in.size) {
+ if (feof(cookie->f))
+ break;
+
+ return -1;
+ }
+
+ cookie->in.pos = 0;
+ }
+
+ r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
+ if (ZSTD_isError(r))
+ return -1;
+ }
+
+ return cookie->out.pos;
+}
+
+static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
+ struct zstd_cookie* cookie = data;
+ int r;
+
+ if (!cookie)
+ return -1;
+
+ // Do not write when mode is "r"
+ if (cookie->mode == 'r')
+ return -1;
+
+ // Configure input buffer
+ cookie->in.src = buffer;
+ cookie->in.size = size;
+ cookie->in.pos = 0;
+
+ while (cookie->in.pos < cookie->in.size) {
+ cookie->out.dst = cookie->buffer;
+ cookie->out.size = sizeof(cookie->buffer);
+ cookie->out.pos = 0;
+
+ r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
+ if (ZSTD_isError(r))
+ return -1;
+
+ r = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
+ if (r < (ssize_t)cookie->out.pos)
+ return -1;
+ }
+
+ return size;
+}
+
+static int zstd_flush(void* data) {
+ struct zstd_cookie* cookie = data;
+ ssize_t bytes_left;
+ int r;
+
+ // Fail if were given no cookie
+ if (!cookie)
+ return -EINVAL;
+
+ // Reset the input buffer
+ cookie->in.src = NULL;
+ cookie->in.size = 0;
+ cookie->in.pos = 0;
+
+ // In write mode, we will have to flush all buffers
+ if (cookie->mode == 'w') {
+ for (;;) {
+ // Reset output buffer
+ cookie->out.dst = cookie->buffer;
+ cookie->out.size = sizeof(cookie->buffer);
+ cookie->out.pos = 0;
+
+ bytes_left = ZSTD_endStream(cookie->cstream, &cookie->out);
+ if (ZSTD_isError(bytes_left))
+ return -1;
+
+ // Otherwise we write the buffer to the file
+ if (cookie->out.pos) {
+ r = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
+ if (r < (ssize_t)cookie->out.pos)
+ return -1;
+ }
+
+ // If the buffer is empty we are done
+ if (bytes_left == 0)
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static int zstd_close(void* data) {
+ struct zstd_cookie* cookie = data;
+ int r;
+
+ // Fail if were given no cookie
+ if (!cookie)
+ return -EINVAL;
+
+ // In write mode, we will have to flush all buffers
+ if (cookie->mode == 'w')
+ zstd_flush(cookie);
+
+ // Close the underlying file handle
+ r = fclose(cookie->f);
+
+ // Free everything
+ zstd_free(cookie);
+
+ return r;
+}
+
+static cookie_io_functions_t zstd_functions = {
+ .read = zstd_read,
+ .write = zstd_write,
+ .seek = NULL,
+ .close = zstd_close,
+};
+
+FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
+ struct zstd_cookie* cookie = NULL;
+ int level = ZSTD_COMPRESSION_LEVEL;
+ int r;
+
+ // Check for a valid file handle
+ if (!f) {
+ errno = EBADFD;
+ return NULL;
+ }
+
+ // Check for some mode
+ if (!mode) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ // Allocate a new cookie
+ cookie = calloc(1, sizeof(*cookie));
+ if (!cookie)
+ goto ERROR;
+
+ // Store the file handle
+ cookie->f = f;
+
+ // Store the mode
+ cookie->mode = *mode;
+
+ switch (cookie->mode) {
+ case 'r':
+ // Allocate stream
+ cookie->dstream = ZSTD_createDStream();
+ if (!cookie->dstream)
+ goto ERROR;
+
+ // Initialize stream
+ r = ZSTD_initDStream(cookie->dstream);
+ if (ZSTD_isError(r))
+ goto ERROR;
+
+ cookie->in.src = cookie->buffer;
+ cookie->in.pos = 0;
+ cookie->in.size = 0;
+ break;
+
+ case 'w':
+ // Try parsing the level
+ r = parse_level(&level, mode);
+ if (r < 0)
+ goto ERROR;
+
+ // Allocate stream
+ cookie->cstream = ZSTD_createCStream();
+ if (!cookie->cstream)
+ goto ERROR;
+
+ // Initialize stream
+ r = ZSTD_initCStream(cookie->cstream, level);
+ if (ZSTD_isError(r))
+ goto ERROR;
+
+ cookie->out.dst = cookie->buffer;
+ cookie->out.pos = 0;
+ cookie->out.size = sizeof(cookie->buffer);
+ break;
+
+ default:
+ errno = ENOTSUP;
+ goto ERROR;
+ }
+
+ return fopencookie(cookie, mode, zstd_functions);
+
+ERROR:
+ if (cookie)
+ zstd_free(cookie);
+
+ return NULL;
+}
--- /dev/null
+/*#############################################################################
+# #
+# Pakfire - The IPFire package management system #
+# Copyright (C) 2025 Pakfire development team #
+# #
+# This program is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# This program is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with this program. If not, see <http://www.gnu.org/licenses/>. #
+# #
+#############################################################################*/
+
+#ifndef PAKFIRE_XFOPEN_H
+#define PAKFIRE_XFOPEN_H
+
+#include <stdio.h>
+
+// Automatically detect
+FILE* pakfire_xfopen(FILE* f, const char* mode);
+
+// gzip
+FILE* pakfire_gzfopen(FILE* f, const char* mode);
+
+// XZ
+FILE* pakfire_xzfopen(FILE* f, const char* mode);
+
+// ZSTD
+FILE* pakfire_zstdfopen(FILE* f, const char* mode);
+
+#endif /* PAKFIRE_XFOPEN_H */
#include <stdio.h>
#include <string.h>
-#include <pakfire/compress.h>
#include <pakfire/string.h>
#include <pakfire/util.h>
+#include <pakfire/xfopen.h>
#include "../testsuite.h"
projects / pakfire.git / commitdiff
