compress: Support reading ZSTD-compressed files

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/Makefile.am b/Makefile.am
index 8dddca705fc43e627c2497962f8ce3dfc99395dd..3d20e4bad7e085d2cae409513dcd4aa367a22bc5 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -423,7 +423,8 @@ tests_libpakfire_compress_CPPFLAGS = \
 tests_libpakfire_compress_LDADD = \
        $(TESTSUITE_LDADD) \
        $(PAKFIRE_LIBS) \
-       $(LZMA_LIBS)
+       $(LZMA_LIBS) \
+       $(ZSTD_LIBS)
 
 tests_libpakfire_db_SOURCES = \
        tests/libpakfire/db.c
@@ -703,6 +704,7 @@ EXTRA_DIST += \
        tests/data/kernel.nm \
        \
        tests/data/compress/data.xz \
+       tests/data/compress/data.zst \
        \
        tests/data/parser/test-comments.txt \
        tests/data/parser/test-conditionals.txt \

diff --git a/src/libpakfire/compress.c b/src/libpakfire/compress.c
index 1160b624db88229dcc289b1d42240e71893eb89d..a1879902982baf5785d573e8a039b1f51f95cd90 100644 (file)
--- a/src/libpakfire/compress.c
+++ b/src/libpakfire/compress.c
@@ -24,6 +24,7 @@
 #include <string.h>
 
 #include <lzma.h>
+#include <zstd.h>
 
 #include <pakfire/compress.h>
 
@@ -36,6 +37,9 @@
 // Settings for XZ
 #define XZ_COMPRESSION_LEVEL   6
 
+// Settings for ZSTD
+#define ZSTD_COMPRESSION_LEVEL 7
+
 const struct compressor {
        char magic[MAX_MAGIC_LENGTH];
        size_t magic_length;
@@ -289,3 +293,158 @@ FILE* pakfire_xzfopen(FILE* f, const char* mode) {
 
        return fopencookie(cookie, mode, xz_functions);
 }
+
+// ZSTD
+
+struct zstd_cookie {
+       FILE* f;
+       char mode;
+       int done;
+
+       // Encoder
+       ZSTD_CStream* cstream;
+       ZSTD_inBuffer in;
+
+       // Decoder
+       ZSTD_DStream* dstream;
+       ZSTD_outBuffer out;
+
+       uint8_t buffer[BUFFER_SIZE];
+};
+
+static ssize_t zstd_read(void* data, char* buffer, size_t size) {
+       struct zstd_cookie* cookie = (struct zstd_cookie*)data;
+       if (!cookie)
+               return -1;
+
+       // Do not read when mode is "w"
+       if (cookie->mode == 'w')
+               return -1;
+
+       if (cookie->done)
+               return 0;
+
+       size_t r = 0;
+
+       // Configure output buffer
+       cookie->out.dst = buffer;
+       cookie->out.pos = 0;
+       cookie->out.size = size;
+
+       while (1) {
+               if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
+                       cookie->in.pos = 0;
+                       cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
+               }
+
+               if (r || cookie->in.size)
+                       r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
+
+               if (r == 0 && feof(cookie->f)) {
+                       cookie->done = 1;
+                       return cookie->out.pos;
+               }
+
+               if (ZSTD_isError(r))
+                       return -1;
+
+               // Buffer full
+               if (cookie->out.pos == size)
+                       return size;
+       }
+}
+
+static int zstd_close(void* data) {
+       struct zstd_cookie* cookie = (struct zstd_cookie*)data;
+       if (!cookie)
+               return -1;
+
+       // XXX handle write
+
+       int r = fclose(cookie->f);
+
+       // Free everything
+       if (cookie->cstream)
+               ZSTD_freeCStream(cookie->cstream);
+       if (cookie->dstream)
+               ZSTD_freeDStream(cookie->dstream);
+       free(cookie);
+
+       return r;
+}
+
+static cookie_io_functions_t zstd_functions = {
+       .read  = zstd_read,
+       //.write = zstd_write,
+       .seek  = NULL,
+       .close = zstd_close,
+};
+
+FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
+       if (!f) {
+               errno = EBADFD;
+               return NULL;
+       }
+
+       if (!mode) {
+               errno = EINVAL;
+               return NULL;
+       }
+
+       struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
+       if (!cookie)
+               return NULL;
+
+       cookie->f = f;
+       cookie->mode = *mode;
+
+       size_t r;
+       switch (cookie->mode) {
+               case 'r':
+                       // Allocate stream
+                       cookie->dstream = ZSTD_createDStream();
+                       if (!cookie->dstream)
+                               goto ERROR;
+
+                       // Initialize stream
+                       r = ZSTD_initDStream(cookie->dstream);
+                       if (ZSTD_isError(r))
+                               goto ERROR;
+
+                       cookie->in.src = cookie->buffer;
+                       cookie->in.pos = 0;
+                       cookie->in.size = 0;
+                       break;
+
+               case 'w':
+                       // Allocate stream
+                       cookie->cstream = ZSTD_createCStream();
+                       if (!cookie->cstream)
+                               goto ERROR;
+
+                       // Initialize stream
+                       r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
+                       if (ZSTD_isError(r))
+                               goto ERROR;
+
+                       cookie->out.dst = cookie->buffer;
+                       cookie->out.pos = 0;
+                       cookie->out.size = sizeof(cookie->buffer);
+                       break;
+
+               default:
+                       errno = ENOTSUP;
+                       goto ERROR;
+       }
+
+       return fopencookie(cookie, mode, zstd_functions);
+
+ERROR:
+       if (cookie->cstream)
+               ZSTD_freeCStream(cookie->cstream);
+       if (cookie->dstream)
+               ZSTD_freeDStream(cookie->dstream);
+       free(cookie);
+
+       return NULL;
+}

diff --git a/src/libpakfire/include/pakfire/compress.h b/src/libpakfire/include/pakfire/compress.h
index 5e6e28cc6651c7509f203608927c216cbb574e2c..633ad29979a90a247fbd9f0655008e0dfd93b3da 100644 (file)
--- a/src/libpakfire/include/pakfire/compress.h
+++ b/src/libpakfire/include/pakfire/compress.h
@@ -29,6 +29,9 @@ FILE* pakfire_xfopen(FILE* f, const char* mode);
 // XZ
 FILE* pakfire_xzfopen(FILE* f, const char* mode);
 
+// ZSTD
+FILE* pakfire_zstdfopen(FILE* f, const char* mode);
+
 #endif
 
 #endif /* PAKFIRE_COMPRESS_H */

diff --git a/tests/data/compress/data.zst b/tests/data/compress/data.zst
new file mode 100644 (file)
index 0000000..2a99d03
Binary files /dev/null and b/tests/data/compress/data.zst differ

diff --git a/tests/libpakfire/compress.c b/tests/libpakfire/compress.c
index fa6ddf6a87b3cd7a5f85ee278373bcb03f972315..36132b97436065660b00bc38322c72a7f688d365 100644 (file)
--- a/tests/libpakfire/compress.c
+++ b/tests/libpakfire/compress.c
@@ -79,14 +79,22 @@ static int test_xzfopen_write(const struct test* t) {
        return EXIT_SUCCESS;
 }
 
+static int test_zstdfopen_read(const struct test* t) {
+       return read_test(t, pakfire_zstdfopen, "data/compress/data.zst");
+}
+
 static int test_xfopen(const struct test* t) {
        return read_test(t, pakfire_xfopen, "data/compress/data.xz");
 }
 
 int main(int argc, char** argv) {
+       // XZ
        testsuite_add_test(test_xzfopen_read);
        testsuite_add_test(test_xzfopen_write);
 
+       // ZSTD
+       testsuite_add_test(test_zstdfopen_read);
+
        testsuite_add_test(test_xfopen);
 
        return testsuite_run();