From: Michael Tremer Date: Fri, 19 Mar 2021 17:09:54 +0000 (+0000) Subject: compress: Support reading ZSTD-compressed files X-Git-Tag: 0.9.28~1285^2~508 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=9476d50248c878c70067fd0d3d5f6268f9aad75d;p=pakfire.git compress: Support reading ZSTD-compressed files Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index 8dddca705..3d20e4bad 100644 --- a/Makefile.am +++ b/Makefile.am @@ -423,7 +423,8 @@ tests_libpakfire_compress_CPPFLAGS = \ tests_libpakfire_compress_LDADD = \ $(TESTSUITE_LDADD) \ $(PAKFIRE_LIBS) \ - $(LZMA_LIBS) + $(LZMA_LIBS) \ + $(ZSTD_LIBS) tests_libpakfire_db_SOURCES = \ tests/libpakfire/db.c @@ -703,6 +704,7 @@ EXTRA_DIST += \ tests/data/kernel.nm \ \ tests/data/compress/data.xz \ + tests/data/compress/data.zst \ \ tests/data/parser/test-comments.txt \ tests/data/parser/test-conditionals.txt \ diff --git a/src/libpakfire/compress.c b/src/libpakfire/compress.c index 1160b624d..a18799029 100644 --- a/src/libpakfire/compress.c +++ b/src/libpakfire/compress.c @@ -24,6 +24,7 @@ #include #include +#include #include @@ -36,6 +37,9 @@ // Settings for XZ #define XZ_COMPRESSION_LEVEL 6 +// Settings for ZSTD +#define ZSTD_COMPRESSION_LEVEL 7 + const struct compressor { char magic[MAX_MAGIC_LENGTH]; size_t magic_length; @@ -289,3 +293,158 @@ FILE* pakfire_xzfopen(FILE* f, const char* mode) { return fopencookie(cookie, mode, xz_functions); } + +// ZSTD + +struct zstd_cookie { + FILE* f; + char mode; + int done; + + // Encoder + ZSTD_CStream* cstream; + ZSTD_inBuffer in; + + // Decoder + ZSTD_DStream* dstream; + ZSTD_outBuffer out; + + uint8_t buffer[BUFFER_SIZE]; +}; + +static ssize_t zstd_read(void* data, char* buffer, size_t size) { + struct zstd_cookie* cookie = (struct zstd_cookie*)data; + if (!cookie) + return -1; + + // Do not read when mode is "w" + if (cookie->mode == 'w') + return -1; + + if (cookie->done) + return 0; + + size_t r = 0; + + // Configure output buffer + cookie->out.dst = buffer; + cookie->out.pos = 0; + cookie->out.size = size; + + while (1) { + if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) { + cookie->in.pos = 0; + cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f); + } + + if (r || cookie->in.size) + r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in); + + if (r == 0 && feof(cookie->f)) { + cookie->done = 1; + return cookie->out.pos; + } + + if (ZSTD_isError(r)) + return -1; + + // Buffer full + if (cookie->out.pos == size) + return size; + } +} + +static int zstd_close(void* data) { + struct zstd_cookie* cookie = (struct zstd_cookie*)data; + if (!cookie) + return -1; + + // XXX handle write + + int r = fclose(cookie->f); + + // Free everything + if (cookie->cstream) + ZSTD_freeCStream(cookie->cstream); + if (cookie->dstream) + ZSTD_freeDStream(cookie->dstream); + free(cookie); + + return r; +} + +static cookie_io_functions_t zstd_functions = { + .read = zstd_read, + //.write = zstd_write, + .seek = NULL, + .close = zstd_close, +}; + +FILE* pakfire_zstdfopen(FILE* f, const char* mode) { + if (!f) { + errno = EBADFD; + return NULL; + } + + if (!mode) { + errno = EINVAL; + return NULL; + } + + struct zstd_cookie* cookie = calloc(1, sizeof(*cookie)); + if (!cookie) + return NULL; + + cookie->f = f; + cookie->mode = *mode; + + size_t r; + switch (cookie->mode) { + case 'r': + // Allocate stream + cookie->dstream = ZSTD_createDStream(); + if (!cookie->dstream) + goto ERROR; + + // Initialize stream + r = ZSTD_initDStream(cookie->dstream); + if (ZSTD_isError(r)) + goto ERROR; + + cookie->in.src = cookie->buffer; + cookie->in.pos = 0; + cookie->in.size = 0; + break; + + case 'w': + // Allocate stream + cookie->cstream = ZSTD_createCStream(); + if (!cookie->cstream) + goto ERROR; + + // Initialize stream + r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL); + if (ZSTD_isError(r)) + goto ERROR; + + cookie->out.dst = cookie->buffer; + cookie->out.pos = 0; + cookie->out.size = sizeof(cookie->buffer); + break; + + default: + errno = ENOTSUP; + goto ERROR; + } + + return fopencookie(cookie, mode, zstd_functions); + +ERROR: + if (cookie->cstream) + ZSTD_freeCStream(cookie->cstream); + if (cookie->dstream) + ZSTD_freeDStream(cookie->dstream); + free(cookie); + + return NULL; +} diff --git a/src/libpakfire/include/pakfire/compress.h b/src/libpakfire/include/pakfire/compress.h index 5e6e28cc6..633ad2997 100644 --- a/src/libpakfire/include/pakfire/compress.h +++ b/src/libpakfire/include/pakfire/compress.h @@ -29,6 +29,9 @@ FILE* pakfire_xfopen(FILE* f, const char* mode); // XZ FILE* pakfire_xzfopen(FILE* f, const char* mode); +// ZSTD +FILE* pakfire_zstdfopen(FILE* f, const char* mode); + #endif #endif /* PAKFIRE_COMPRESS_H */ diff --git a/tests/data/compress/data.zst b/tests/data/compress/data.zst new file mode 100644 index 000000000..2a99d033e Binary files /dev/null and b/tests/data/compress/data.zst differ diff --git a/tests/libpakfire/compress.c b/tests/libpakfire/compress.c index fa6ddf6a8..36132b974 100644 --- a/tests/libpakfire/compress.c +++ b/tests/libpakfire/compress.c @@ -79,14 +79,22 @@ static int test_xzfopen_write(const struct test* t) { return EXIT_SUCCESS; } +static int test_zstdfopen_read(const struct test* t) { + return read_test(t, pakfire_zstdfopen, "data/compress/data.zst"); +} + static int test_xfopen(const struct test* t) { return read_test(t, pakfire_xfopen, "data/compress/data.xz"); } int main(int argc, char** argv) { + // XZ testsuite_add_test(test_xzfopen_read); testsuite_add_test(test_xzfopen_write); + // ZSTD + testsuite_add_test(test_zstdfopen_read); + testsuite_add_test(test_xfopen); return testsuite_run();