From: Michael Schroeder <mls@suse.de>
Date: Mon, 9 Jul 2018 09:34:10 +0000 (+0200)
Subject: Support zstd compression
X-Git-Tag: 0.6.35~41
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f51d048e6774c180a9fef21b87d7303accae3809;p=thirdparty%2Flibsolv.git

Support zstd compression
---

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4b10f0ce..f4678af6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -33,6 +33,7 @@ OPTION (MULTI_SEMANTICS "Build with support for multiple distribution types?" OF
 
 OPTION (ENABLE_LZMA_COMPRESSION "Build with lzma/xz compression support?" OFF)
 OPTION (ENABLE_BZIP2_COMPRESSION "Build with bzip2 compression support?" OFF)
+OPTION (ENABLE_ZSTD_COMPRESSION "Build with zstd compression support?" OFF)
 
 OPTION (WITH_LIBXML2  "Build with libxml2 instead of libexpat?" OFF)
 
@@ -176,6 +177,10 @@ IF (ENABLE_BZIP2_COMPRESSION)
 FIND_PACKAGE (BZip2 REQUIRED)
 ENDIF (ENABLE_BZIP2_COMPRESSION)
 
+IF (ENABLE_ZSTD_COMPRESSION)
+FIND_LIBRARY (ZSTD_LIBRARY NAMES zstd)
+ENDIF (ENABLE_ZSTD_COMPRESSION)
+
 IF (RPM5)
 MESSAGE (STATUS "Enabling RPM 5 support")
 ADD_DEFINITIONS (-DRPM5)
@@ -284,7 +289,7 @@ FOREACH (VAR
   ENABLE_SUSEREPO ENABLE_COMPS ENABLE_TESTCASE_HELIXREPO
   ENABLE_HELIXREPO ENABLE_MDKREPO ENABLE_ARCHREPO ENABLE_DEBIAN ENABLE_HAIKU
   ENABLE_ZLIB_COMPRESSION ENABLE_LZMA_COMPRESSION ENABLE_BZIP2_COMPRESSION
-  ENABLE_PGPVRFY ENABLE_APPDATA)
+  ENABLE_ZSTD_COMPRESSION ENABLE_PGPVRFY ENABLE_APPDATA)
   IF(${VAR})
     ADD_DEFINITIONS (-D${VAR}=1)
     SET (SWIG_FLAGS ${SWIG_FLAGS} -D${VAR})
@@ -386,6 +391,9 @@ ENDIF (ENABLE_LZMA_COMPRESSION)
 IF (ENABLE_BZIP2_COMPRESSION)
 SET (SYSTEM_LIBRARIES ${SYSTEM_LIBRARIES} ${BZIP2_LIBRARIES})
 ENDIF (ENABLE_BZIP2_COMPRESSION)
+IF (ENABLE_ZSTD_COMPRESSION)
+SET (SYSTEM_LIBRARIES ${SYSTEM_LIBRARIES} ${ZSTD_LIBRARY})
+ENDIF (ENABLE_ZSTD_COMPRESSION)
 IF (ENABLE_RPMDB)
 SET (SYSTEM_LIBRARIES ${RPMDB_LIBRARY} ${SYSTEM_LIBRARIES})
 ENDIF (ENABLE_RPMDB)
diff --git a/ext/solv_xfopen.c b/ext/solv_xfopen.c
index 2c64bb62..f15bdb8a 100644
--- a/ext/solv_xfopen.c
+++ b/ext/solv_xfopen.c
@@ -316,6 +316,191 @@ static inline FILE *mylzfdopen(int fd, const char *mode)
 
 #endif /* ENABLE_LZMA_COMPRESSION */
 
+#ifdef ENABLE_ZSTD_COMPRESSION
+
+#include <zstd.h>
+
+typedef struct zstdfile {
+  ZSTD_CStream *cstream;
+  ZSTD_DStream *dstream;
+  FILE *file;
+  int encoding;
+  int eof;
+  ZSTD_inBuffer in;
+  ZSTD_outBuffer out;
+  unsigned char buf[1 << 15];
+} ZSTDFILE;
+
+static ZSTDFILE *zstdopen(const char *path, const char *mode, int fd)
+{
+  int level = 7;
+  int encoding = 0;
+  FILE *fp;
+  ZSTDFILE *zstdfile;
+
+  if (!path && fd < 0)
+    return 0;
+  for (; *mode; mode++)
+    {
+      if (*mode == 'w')
+	encoding = 1;
+      else if (*mode == 'r')
+	encoding = 0;
+      else if (*mode >= '1' && *mode <= '9')
+	level = *mode - '0';
+    }
+  if (fd != -1)
+    fp = fdopen(fd, encoding ? "w" : "r");
+  else
+    fp = fopen(path, encoding ? "w" : "r");
+  if (!fp)
+    return 0;
+  zstdfile = solv_calloc(1, sizeof(*zstdfile));
+  zstdfile->encoding = encoding;
+  if (encoding)
+    {
+      zstdfile->cstream = ZSTD_createCStream();
+      zstdfile->encoding = 1;
+      if (!zstdfile->cstream)
+	{
+	  solv_free(zstdfile);
+	  fclose(fp);
+	  return 0;
+	}
+      if (ZSTD_isError(ZSTD_initCStream(zstdfile->cstream, level)))
+ 	{
+	  ZSTD_freeCStream(zstdfile->cstream);
+	  solv_free(zstdfile);
+	  fclose(fp);
+	  return 0;
+	}
+      zstdfile->out.dst = zstdfile->buf;
+      zstdfile->out.pos = 0;
+      zstdfile->out.size = sizeof(zstdfile->buf);
+    }
+  else
+    {
+      zstdfile->dstream = ZSTD_createDStream();
+      if (ZSTD_isError(ZSTD_initDStream(zstdfile->dstream)))
+ 	{
+	  ZSTD_freeDStream(zstdfile->dstream);
+	  solv_free(zstdfile);
+	  fclose(fp);
+	  return 0;
+	}
+      zstdfile->in.src = zstdfile->buf;
+      zstdfile->in.pos = 0;
+      zstdfile->in.size = 0;
+    }
+  zstdfile->file = fp;
+  return zstdfile;
+}
+
+static int zstdclose(void *cookie)
+{
+  ZSTDFILE *zstdfile = cookie;
+  int rc;
+
+  if (!zstdfile)
+    return -1;
+  if (zstdfile->encoding)
+    {
+      for (;;)
+	{
+	  size_t ret;
+	  zstdfile->out.pos = 0;
+	  ret = ZSTD_endStream(zstdfile->cstream, &zstdfile->out);
+	  if (ZSTD_isError(ret))
+	    return -1;
+	  if (zstdfile->out.pos && fwrite(zstdfile->buf, 1, zstdfile->out.pos, zstdfile->file) != zstdfile->out.pos)
+	    return -1;
+	  if (ret == 0)
+	    break;
+	}
+      ZSTD_freeCStream(zstdfile->cstream);
+    }
+  else
+    {
+      ZSTD_freeDStream(zstdfile->dstream);
+    }
+  rc = fclose(zstdfile->file);
+  free(zstdfile);
+  return rc;
+}
+
+static ssize_t zstdread(void *cookie, char *buf, size_t len)
+{
+  ZSTDFILE *zstdfile = cookie;
+  int eof = 0;
+  size_t ret = 0;
+  if (!zstdfile || zstdfile->encoding)
+    return -1;
+  if (zstdfile->eof)
+    return 0;
+  zstdfile->out.dst = buf;
+  zstdfile->out.pos = 0;
+  zstdfile->out.size = len;
+  for (;;)
+    {
+      if (!eof && zstdfile->in.pos == zstdfile->in.size)
+	{
+	  zstdfile->in.pos = 0;
+	  zstdfile->in.size = fread(zstdfile->buf, 1, sizeof(zstdfile->buf), zstdfile->file);
+	  if (!zstdfile->in.size)
+	    eof = 1;
+	}
+      if (ret || !eof)
+        ret = ZSTD_decompressStream(zstdfile->dstream, &zstdfile->out, &zstdfile->in);
+      if (ret == 0 && eof)
+	{
+	  zstdfile->eof = 1;
+	  return zstdfile->out.pos;
+	}
+      if (ZSTD_isError(ret))
+	return -1;
+      if (zstdfile->out.pos == len)
+	return len;
+    }
+}
+
+static ssize_t zstdwrite(void *cookie, const char *buf, size_t len)
+{
+  ZSTDFILE *zstdfile = cookie;
+  if (!zstdfile || !zstdfile->encoding)
+    return -1;
+  if (!len)
+    return 0;
+  zstdfile->in.src = buf;
+  zstdfile->in.pos = 0;
+  zstdfile->in.size = len;
+
+  for (;;)
+    {
+      size_t ret;
+      zstdfile->out.pos = 0;
+      ret = ZSTD_compressStream(zstdfile->cstream, &zstdfile->out, &zstdfile->in);
+      if (ZSTD_isError(ret))
+        return -1;
+      if (zstdfile->out.pos && fwrite(zstdfile->buf, 1, zstdfile->out.pos, zstdfile->file) != zstdfile->out.pos)
+        return -1;
+      if (zstdfile->in.pos == len)
+        return len;
+    }
+}
+
+static inline FILE *myzstdfopen(const char *fn, const char *mode)
+{
+  ZSTDFILE *zstdfile = zstdopen(fn, mode, -1);
+  return cookieopen(zstdfile, mode, zstdread, zstdwrite, zstdclose);
+}
+
+static inline FILE *myzstdfdopen(int fd, const char *mode)
+{
+  ZSTDFILE *zstdfile = zstdopen(0, mode, fd);
+  return cookieopen(zstdfile, mode, zstdread, zstdwrite, zstdclose);
+}
+
+#endif
 
 FILE *
 solv_xfopen(const char *fn, const char *mode)
@@ -351,6 +536,13 @@ solv_xfopen(const char *fn, const char *mode)
 #else
   if (suf && !strcmp(suf, ".bz2"))
     return 0;
+#endif
+#ifdef ENABLE_ZSTD_COMPRESSION
+  if (suf && !strcmp(suf, ".zst"))
+    return myzstdfopen(fn, mode);
+#else
+  if (suf && !strcmp(suf, ".zst"))
+    return 0;
 #endif
   return fopen(fn, mode);
 }
@@ -402,6 +594,13 @@ solv_xfopen_fd(const char *fn, int fd, const char *mode)
 #else
   if (suf && !strcmp(suf, ".bz2"))
     return 0;
+#endif
+#ifdef ENABLE_ZSTD_COMPRESSION
+  if (suf && !strcmp(suf, ".zst"))
+    return myzstdfdopen(fd, simplemode);
+#else
+  if (suf && !strcmp(suf, ".zst"))
+    return 0;
 #endif
   return fdopen(fd, mode);
 }
@@ -429,6 +628,12 @@ solv_xfopen_iscompressed(const char *fn)
     return 1;
 #else
     return -1;
+#endif
+  if (!strcmp(suf, ".zst"))
+#ifdef ENABLE_ZSTD_COMPRESSION
+    return 1;
+#else
+    return -1;
 #endif
   return 0;
 }
diff --git a/src/solvversion.h.in b/src/solvversion.h.in
index 7d107f96..4caba471 100644
--- a/src/solvversion.h.in
+++ b/src/solvversion.h.in
@@ -43,6 +43,7 @@ extern int solv_version_patch;
 #cmakedefine LIBSOLVEXT_FEATURE_ZLIB_COMPRESSION
 #cmakedefine LIBSOLVEXT_FEATURE_LZMA_COMPRESSION
 #cmakedefine LIBSOLVEXT_FEATURE_BZIP2_COMPRESSION
+#cmakedefine LIBSOLVEXT_FEATURE_ZSTD_COMPRESSION
 
 /* see tools/common_write.c for toolversion history */
 #define LIBSOLV_TOOLVERSION "1.1"