zwrapbench: testing reusing of a context

diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index 318a890c0528f24fdac30b9ccda46f71fe19c26d..4659cbb921d2a751fa5a52f4e2e57c85c1a68b2b 100644 (file)
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -138,7 +138,7 @@ typedef struct
     size_t resSize;
 } blockParam_t;
 
-typedef enum { BMK_ZSTD, BMK_ZSTD2, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD } BMK_compressor;
+typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD, BMK_ZLIB_REUSE, BMK_ZWRAP_ZLIB_REUSE, BMK_ZWRAP_ZSTD_REUSE } BMK_compressor;
 
 
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
@@ -249,19 +249,20 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         nbLoops++;
                     } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
                     ZSTD_freeCDict(cdict);
-                } else if (compressor == BMK_ZSTD2) {
+                } else if (compressor == BMK_ZSTD_STREAM) {
                     ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
                     ZSTD_inBuffer inBuffer;
                     ZSTD_outBuffer outBuffer;
                     ZSTD_CStream* zbc = ZSTD_createCStream();
+                    size_t rSize;
                     if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure");
-
+                    rSize = ZSTD_initCStream_advanced(zbc, NULL, 0, zparams, avgSize);
+                    if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize));
                     do {
                         U32 blockNb;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
-                            size_t rSize;
-                            rSize = ZSTD_initCStream_advanced(zbc, NULL, 0, zparams, avgSize);
-                            if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize));
+                            rSize = ZSTD_resetCStream(zbc, avgSize);
+                            if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetCStream() failed : %s", ZSTD_getErrorName(rSize));
                             inBuffer.src = blockTable[blockNb].srcPtr;
                             inBuffer.size = blockTable[blockNb].srcSize;
                             inBuffer.pos = 0;
@@ -277,11 +278,44 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         nbLoops++;
                     } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
                     ZSTD_freeCStream(zbc);
+                } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) {
+                    z_stream def;
+                    int ret;
+                    if (compressor == BMK_ZLIB_REUSE || compressor == BMK_ZWRAP_ZLIB_REUSE) useZSTDcompression(0);
+                    else useZSTDcompression(1);
+                    def.zalloc = Z_NULL;
+                    def.zfree = Z_NULL;
+                    def.opaque = Z_NULL;
+                    ret = deflateInit(&def, cLevel);
+                    if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
+                    if (isUsingZSTDcompression()) {
+                        ret = ZSTD_setPledgedSrcSize(&def, avgSize);
+                        if (ret != Z_OK) EXM_THROW(1, "ZSTD_setPledgedSrcSize failure");
+                    }
+                    do {
+                        U32 blockNb;
+                        for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+                            ret = deflateReset(&def);
+                            if (ret != Z_OK) EXM_THROW(1, "deflateReset failure");
+                            def.next_in = (const void*) blockTable[blockNb].srcPtr;
+                            def.avail_in = blockTable[blockNb].srcSize;
+                            def.total_in = 0;
+                            def.next_out = (void*) blockTable[blockNb].cPtr;
+                            def.avail_out = blockTable[blockNb].cRoom;
+                            def.total_out = 0;
+                            ret = deflate(&def, Z_FINISH);
+                            if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure");
+                            blockTable[blockNb].cSize = def.total_out;
+                        }
+                        nbLoops++;
+                    } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+                    ret = deflateEnd(&def);
+                    if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure");
                 } else {
+                    z_stream def;
                     if (compressor == BMK_ZLIB || compressor == BMK_ZWRAP_ZLIB) useZSTDcompression(0);
                     else useZSTDcompression(1);
                     do {
-                        z_stream def;
                         U32 blockNb;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                             int ret;
@@ -355,18 +389,19 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         nbLoops++;
                     } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
                     ZSTD_freeDDict(ddict);
-                } else if (compressor == BMK_ZSTD2) {
+                } else if (compressor == BMK_ZSTD_STREAM) {
                     ZSTD_inBuffer inBuffer;
                     ZSTD_outBuffer outBuffer;
                     ZSTD_DStream* zbd = ZSTD_createDStream();
+                    size_t rSize;
                     if (zbd == NULL) EXM_THROW(1, "ZSTD_createDStream() allocation failure");
-
+                    rSize = ZSTD_initDStream(zbd);
+                    if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initDStream() failed : %s", ZSTD_getErrorName(rSize));
                     do {
                         U32 blockNb;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
-                            size_t rSize;
-                            rSize = ZSTD_initDStream(zbd);
-                            if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initDStream() failed : %s", ZSTD_getErrorName(rSize));
+                            rSize = ZSTD_resetDStream(zbd);
+                            if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetDStream() failed : %s", ZSTD_getErrorName(rSize));
                             inBuffer.src = blockTable[blockNb].cPtr;
                             inBuffer.size = blockTable[blockNb].cSize;
                             inBuffer.pos = 0;
@@ -380,12 +415,41 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         nbLoops++;
                     } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
                     ZSTD_freeDStream(zbd);
+                } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) {
+                    z_stream inf;
+                    int ret;
+                    if (compressor == BMK_ZLIB_REUSE) setZWRAPdecompressionType(ZWRAP_FORCE_ZLIB);
+                    else setZWRAPdecompressionType(ZWRAP_AUTO);
+                    inf.zalloc = Z_NULL;
+                    inf.zfree = Z_NULL;
+                    inf.opaque = Z_NULL;
+                    ret = inflateInit(&inf);
+                    if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
+                    do {
+                        U32 blockNb;
+                        for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+                            ret = inflateReset(&inf);
+                            if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
+                            inf.next_in = (const void*) blockTable[blockNb].cPtr;
+                            inf.avail_in = blockTable[blockNb].cSize;
+                            inf.total_in = 0;
+                            inf.next_out = (void*) blockTable[blockNb].resPtr;
+                            inf.avail_out = blockTable[blockNb].srcSize;
+                            inf.total_out = 0;
+                            ret = inflate(&inf, Z_FINISH);
+                            if (ret != Z_STREAM_END) EXM_THROW(1, "inflate failure");
+                            blockTable[blockNb].resSize = inf.total_out;
+                        }
+                        nbLoops++;
+                    } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+                    ret = inflateEnd(&inf);
+                    if (ret != Z_OK) EXM_THROW(1, "inflateEnd failure");
                 } else {
+                    z_stream inf;
                     if (compressor == BMK_ZLIB) setZWRAPdecompressionType(ZWRAP_FORCE_ZLIB);
                     else setZWRAPdecompressionType(ZWRAP_AUTO);
                     do {
                         U32 blockNb;
-                        z_stream inf;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                             int ret;
                             inf.zalloc = Z_NULL;
@@ -504,7 +568,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
 
     if (cLevelLast < cLevel) cLevelLast = cLevel;
 
-    DISPLAY("benchmarking zstd %s (ZSTD_CCtx)\n", ZSTD_VERSION_STRING);
+    DISPLAY("benchmarking zstd %s (using ZSTD_CCtx)\n", ZSTD_VERSION_STRING);
     for (l=cLevel; l <= cLevelLast; l++) {
         BMK_benchMem(srcBuffer, benchedSize,
                      displayName, l,
@@ -512,15 +576,15 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
                      dictBuffer, dictBufferSize, BMK_ZSTD);
     }
 
-    DISPLAY("benchmarking zstd %s (ZSTD_CStream)\n", ZSTD_VERSION_STRING);
+    DISPLAY("benchmarking zstd %s (using ZSTD_CStream)\n", ZSTD_VERSION_STRING);
     for (l=cLevel; l <= cLevelLast; l++) {
         BMK_benchMem(srcBuffer, benchedSize,
                      displayName, l,
                      fileSizes, nbFiles,
-                     dictBuffer, dictBufferSize, BMK_ZSTD2);
+                     dictBuffer, dictBufferSize, BMK_ZSTD_STREAM);
     }
 
-    DISPLAY("benchmarking zlibWrapper with zstd %s\n", ZSTD_VERSION_STRING);
+    DISPLAY("benchmarking zstd %s (using zlibWrapper)\n", ZSTD_VERSION_STRING);
     for (l=cLevel; l <= cLevelLast; l++) {
         BMK_benchMem(srcBuffer, benchedSize,
                      displayName, l,
@@ -528,6 +592,14 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
                      dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD);
     }
 
+    DISPLAY("benchmarking zstd %s (zlibWrapper with reusing a context)\n", ZSTD_VERSION_STRING);
+    for (l=cLevel; l <= cLevelLast; l++) {
+        BMK_benchMem(srcBuffer, benchedSize,
+                     displayName, l,
+                     fileSizes, nbFiles,
+                     dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD_REUSE);
+    }
+
 
     if (cLevelLast > Z_BEST_COMPRESSION) cLevelLast = Z_BEST_COMPRESSION;
 
@@ -539,13 +611,29 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
                      dictBuffer, dictBufferSize, BMK_ZLIB);
     }
 
-    DISPLAY("benchmarking zlibWrapper with zlib %s\n", ZLIB_VERSION);
+    DISPLAY("benchmarking zlib %s (reusing a context)\n", ZLIB_VERSION);
+    for (l=cLevel; l <= cLevelLast; l++) {
+        BMK_benchMem(srcBuffer, benchedSize,
+                     displayName, l,
+                     fileSizes, nbFiles,
+                     dictBuffer, dictBufferSize, BMK_ZLIB_REUSE);
+    }
+
+    DISPLAY("benchmarking zlib %s (using zlibWrapper)\n", ZLIB_VERSION);
     for (l=cLevel; l <= cLevelLast; l++) {
         BMK_benchMem(srcBuffer, benchedSize,
                      displayName, l,
                      fileSizes, nbFiles,
                      dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB);
     }
+
+    DISPLAY("benchmarking zlib %s (zlibWrapper with reusing a context)\n", ZLIB_VERSION);
+    for (l=cLevel; l <= cLevelLast; l++) {
+        BMK_benchMem(srcBuffer, benchedSize,
+                     displayName, l,
+                     fileSizes, nbFiles,
+                     dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB_REUSE);
+    }
 }
 
 

diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 11ba941802f498efddad367f59b4aac5accb528f..c9a04b8b0e264aaa2d6d3d41cd9d8943a5887746 100644 (file)
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -425,7 +425,7 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
 size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
 {
     if (zwd==NULL) return 0;   /* support free on null */
-    ZSTD_freeDStream(zwd->zbd);
+    if (zwd->zbd) ZSTD_freeDStream(zwd->zbd);
     if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version);
     zwd->customMem.customFree(zwd->customMem.opaque, zwd);
     return 0;
@@ -505,8 +505,10 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
 
     {   ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
         if (zwd == NULL) return Z_STREAM_ERROR;
-        { size_t const errorCode = ZSTD_resetDStream(zwd->zbd);
-          if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); }
+        if (zwd->zbd) { 
+            size_t const errorCode = ZSTD_resetDStream(zwd->zbd);
+            if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); 
+        }
         ZWRAP_initDCtx(zwd);
     }
     
@@ -545,7 +547,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
 
     {   size_t errorCode;
         ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
-        if (zwd == NULL) return Z_STREAM_ERROR;
+        if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
         errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
         if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
 
@@ -580,17 +582,15 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
     }
 
     if (strm->avail_in > 0) {
-        size_t errorCode, srcSize, inPos;
+        size_t errorCode, srcSize;
         ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
         if (zwd == NULL) return Z_STREAM_ERROR;
         LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
 
         if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END;
 
-     //   if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZLIB_HEADERSIZE))
         if (strm->total_in < ZLIB_HEADERSIZE)
         {
-          //  printf(".");
             srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in);
             memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
             strm->total_in += srcSize;
@@ -637,10 +637,13 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
             }
         }
 
-     //   if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZSTD_HEADERSIZE))
+        if (!zwd->zbd) {
+            zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
+            if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
+        }
+
         if (strm->total_in < ZSTD_HEADERSIZE)
         {
-          //  printf("+");
             srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
             memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
             strm->total_in += srcSize;
@@ -648,15 +651,11 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
             strm->avail_in -= srcSize;
             if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK;
 
-            zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
-            if (zwd->zbd == NULL) goto error;
-
             errorCode = ZSTD_initDStream(zwd->zbd);
-            if (ZSTD_isError(errorCode)) goto error;
+            if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
 
             if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
 
-            inPos = zwd->inBuffer.pos;
             zwd->inBuffer.src = zwd->headerBuf;
             zwd->inBuffer.size = ZSTD_HEADERSIZE;
             zwd->inBuffer.pos = 0;
@@ -669,11 +668,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                 LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
                 goto error;
             }
-          //  LOG_WRAPPERD("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos);
             if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */
         }
 
-        inPos = 0;//zwd->inBuffer.pos;
         zwd->inBuffer.src = strm->next_in;
         zwd->inBuffer.size = strm->avail_in;
         zwd->inBuffer.pos = 0;
@@ -687,13 +684,13 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
             LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n", ZSTD_getErrorName(errorCode), zwd->errorCount);
             if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error;
         }
-        LOG_WRAPPERD("inflate inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size);
+        LOG_WRAPPERD("inflate inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size);
         strm->next_out += zwd->outBuffer.pos;
         strm->total_out += zwd->outBuffer.pos;
         strm->avail_out -= zwd->outBuffer.pos;
-        strm->total_in += zwd->inBuffer.pos - inPos;
-        strm->next_in += zwd->inBuffer.pos - inPos;
-        strm->avail_in -= zwd->inBuffer.pos - inPos;
+        strm->total_in += zwd->inBuffer.pos;
+        strm->next_in += zwd->inBuffer.pos;
+        strm->avail_in -= zwd->inBuffer.pos;
         if (errorCode == 0) { 
             LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); 
             zwd->decompState = Z_STREAM_END;