faster inflate() autodetection of zlib/zstd

diff --git a/.travis.yml b/.travis.yml
index be8a90afeb64421c3cb1ca126c9287df5c2acc04..58728d89cd5ff09d0b5c0670403c864ac04111a7 100644 (file)
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,9 +3,63 @@ compiler: gcc
 matrix:
   fast_finish: true
   include:
+    # OS X Mavericks
+    - os: osx
+      env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install"
+    # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
     - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install"
+    - os: linux
+      sudo: false
+      language: cpp
+      install:
+        - export CXX="g++-4.8" CC="gcc-4.8"
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - gcc-4.8
+            - g++-4.8
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean"
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
+    # Standard Ubuntu 12.04 LTS Server Edition 64 bit
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" CMD="make armtest"
+      addons:
+        apt:
+          packages:
+            - gcc-arm-linux-gnueabi
+            - libc6-dev-armel-cross
+            - linux-libc-dev-armel-cross
+            - binfmt-support
+            - qemu
+            - qemu-user-static
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" CMD="make asan32"
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - libc6-dev-i386
+            - gcc-multilib
+    # Ubuntu 14.04 LTS Server Edition 64 bit
+    - os: linux
+      dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest'
+      env: PLATFORM="Ubuntu 14.04" CMD="make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest"
       addons:
         apt:
           packages:
@@ -13,7 +67,25 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
+      env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest"
+      addons:
+        apt:
+          packages:
+            - libc6-dev-i386
+            - g++-multilib 
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make -C tests test32"
+      addons:
+        apt:
+          packages:
+            - libc6-dev-i386
+            - gcc-multilib
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest"
       addons:
         apt:
           sources:

diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index a612f2593fa1544aaad8d1871e2bcb3b174c6bb5..ea025b951e598a95712c007e9a4a2a20a17d25ef 100644 (file)
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -1,8 +1,8 @@
 # Makefile for example of using zstd wrapper for zlib
 #
-# make - compiles statically and dynamically linked examples
-# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples with zstd compression turned on
-# make test test_d - runs statically and dynamically linked examples
+# make - compiles examples
+# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
+# make test - runs examples
 
 
 # Paths to static and dynamic zlib and zstd libraries
@@ -22,7 +22,7 @@ LDFLAGS = $(LOC)
 RM = rm -f
 
 
-all: clean fitblk example example_d zwrapbench
+all: clean fitblk example zwrapbench
 
 test: example fitblk example_zstd fitblk_zstd zwrapbench
        ./example
@@ -34,11 +34,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench
        ./zwrapbench -qb1e5 ../zstd_compression_format.md
        ./zwrapbench -qb1e5B1K ../zstd_compression_format.md
 
-test_d: example_d
-       ./example_d
-
+#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
 valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
-valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
 valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
        @echo "\n ---- valgrind tests ----"
        $(VALGRIND) ./example
@@ -83,5 +80,5 @@ $(ZSTDLIBDIR)/libzstd.so:
        $(MAKE) -C $(ZSTDLIBDIR) all
 
 clean:
-       -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd zwrapbench
+       -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench
        @echo Cleaning completed

diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 7a825899c6945d597fe62bd28ee48f7c2c24be10..76f075108d3f270794812e29a7027104e129a0ad 100644 (file)
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -548,6 +548,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
         if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
         errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
         if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
+        zwd->decompState = Z_NEED_DICT; 
 
         if (strm->total_in == ZSTD_HEADERSIZE) {
             zwd->inBuffer.src = zwd->headerBuf;
@@ -571,6 +572,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
 
 ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
 {
+    ZWRAP_DCtx* zwd;
     int res;
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) {
         LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
@@ -581,60 +583,79 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
 
     if (strm->avail_in > 0) {
         size_t errorCode, srcSize;
-        ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
-        if (zwd == NULL) return Z_STREAM_ERROR;
+        zwd = (ZWRAP_DCtx*) strm->state;
         LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
 
+        if (zwd == NULL) return Z_STREAM_ERROR;
         if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END;
 
-        if (strm->total_in < ZLIB_HEADERSIZE)
-        {
-            srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in);
-            memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
-            strm->total_in += srcSize;
-            strm->next_in += srcSize;
-            strm->avail_in -= srcSize;
-            if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK;
-
-            if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
-                z_stream strm2;
-                strm2.next_in = strm->next_in;
-                strm2.avail_in = strm->avail_in;
-                strm2.next_out = strm->next_out;
-                strm2.avail_out = strm->avail_out;
-
-                if (zwd->windowBits)
-                    errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
-                else
-                    errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
-                LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
-                if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
-
-                /* inflate header */
-                strm->next_in = (unsigned char*)zwd->headerBuf;
-                strm->avail_in = ZLIB_HEADERSIZE;
-                strm->avail_out = 0;
-                errorCode = inflate(strm, Z_NO_FLUSH);
-                LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
-                if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
-                if (strm->avail_in > 0) goto error;
-
-                strm->next_in = strm2.next_in;
-                strm->avail_in = strm2.avail_in;
-                strm->next_out = strm2.next_out;
-                strm->avail_out = strm2.avail_out;
-
-                strm->reserved = 0; /* mark as zlib stream */
-                errorCode = ZWRAP_freeDCtx(zwd);
-                if (ZSTD_isError(errorCode)) goto error;
-
-                if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
-                else res = inflate(strm, flush);
-                LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
-                return res;
+        if (strm->total_in < ZLIB_HEADERSIZE) {
+            if (strm->total_in == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
+                if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
+                    if (zwd->windowBits)
+                        errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
+                    else
+                        errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
+
+                    strm->reserved = 0; /* mark as zlib stream */
+                    errorCode = ZWRAP_freeDCtx(zwd);
+                    if (ZSTD_isError(errorCode)) goto error;
+
+                    if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
+                    else res = inflate(strm, flush);
+                    LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
+                    return res;
+                }
+            } else {
+                srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in);
+                memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
+                strm->total_in += srcSize;
+                strm->next_in += srcSize;
+                strm->avail_in -= srcSize;
+                if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK;
+
+                if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
+                    z_stream strm2;
+                    strm2.next_in = strm->next_in;
+                    strm2.avail_in = strm->avail_in;
+                    strm2.next_out = strm->next_out;
+                    strm2.avail_out = strm->avail_out;
+
+                    if (zwd->windowBits)
+                        errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
+                    else
+                        errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
+                    LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
+                    if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
+
+                    /* inflate header */
+                    strm->next_in = (unsigned char*)zwd->headerBuf;
+                    strm->avail_in = ZLIB_HEADERSIZE;
+                    strm->avail_out = 0;
+                    errorCode = inflate(strm, Z_NO_FLUSH);
+                    LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
+                    if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
+                    if (strm->avail_in > 0) goto error;
+
+                    strm->next_in = strm2.next_in;
+                    strm->avail_in = strm2.avail_in;
+                    strm->next_out = strm2.next_out;
+                    strm->avail_out = strm2.avail_out;
+
+                    strm->reserved = 0; /* mark as zlib stream */
+                    errorCode = ZWRAP_freeDCtx(zwd);
+                    if (ZSTD_isError(errorCode)) goto error;
+
+                    if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
+                    else res = inflate(strm, flush);
+                    LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
+                    return res;
+                }
             }
         }
 
+        if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
+
         if (!zwd->zbd) {
             zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
             if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
@@ -642,31 +663,36 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
 
         if (strm->total_in < ZSTD_HEADERSIZE)
         {
-            srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
-            memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
-            strm->total_in += srcSize;
-            strm->next_in += srcSize;
-            strm->avail_in -= srcSize;
-            if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK;
-
-            errorCode = ZSTD_initDStream(zwd->zbd);
-            if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
-
-            if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
-
-            zwd->inBuffer.src = zwd->headerBuf;
-            zwd->inBuffer.size = ZSTD_HEADERSIZE;
-            zwd->inBuffer.pos = 0;
-            zwd->outBuffer.dst = strm->next_out;
-            zwd->outBuffer.size = 0;
-            zwd->outBuffer.pos = 0;
-            errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
-            LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
-            if (ZSTD_isError(errorCode)) {
-                LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
-                goto error;
+            if (strm->total_in == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
+                if (zwd->decompState != Z_NEED_DICT) {
+                    errorCode = ZSTD_initDStream(zwd->zbd);
+                    if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
+                }
+            } else {
+                srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
+                memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
+                strm->total_in += srcSize;
+                strm->next_in += srcSize;
+                strm->avail_in -= srcSize;
+                if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK;
+
+                errorCode = ZSTD_initDStream(zwd->zbd);
+                if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
+
+                zwd->inBuffer.src = zwd->headerBuf;
+                zwd->inBuffer.size = ZSTD_HEADERSIZE;
+                zwd->inBuffer.pos = 0;
+                zwd->outBuffer.dst = strm->next_out;
+                zwd->outBuffer.size = 0;
+                zwd->outBuffer.pos = 0;
+                errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
+                LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
+                if (ZSTD_isError(errorCode)) {
+                    LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
+                    goto error;
+                }
+                if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */
             }
-            if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */
         }
 
         zwd->inBuffer.src = strm->next_in;
@@ -694,13 +720,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
             zwd->decompState = Z_STREAM_END; 
             return Z_STREAM_END;
         }
-        goto finish;
-error:
-        return ZWRAPD_finishWithError(zwd, strm, 0);
     }
-finish:
     LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
     return Z_OK;
+
+error:
+    return ZWRAPD_finishWithError(zwd, strm, 0);
 }