CLI : automatically set overlap size to max (windowSize) for max compression level

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 988e133d6f65fc93c2d37848a7c4810ce4bc5d0b..5f0bf2ab15f864d04ba648940b175626612c2256 100644 (file)
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -285,6 +285,7 @@ struct ZSTDMT_CCtx_s {
     unsigned nextJobID;
     unsigned frameEnded;
     unsigned allJobsCompleted;
+    unsigned overlapWrLog;
     unsigned long long frameContentSize;
     size_t sectionSize;
     ZSTD_CDict* cdict;
@@ -298,7 +299,6 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
     U32 const minNbJobs = nbThreads + 2;
     U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1;
     U32 const nbJobs = 1 << nbJobsLog2;
-    //nbThreads = 1;   /* for tests */
     DEBUGLOG(5, "nbThreads : %u  ; minNbJobs : %u ;  nbJobsLog2 : %u ;  nbJobs : %u  \n",
             nbThreads, minNbJobs, nbJobsLog2, nbJobs);
     if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
@@ -308,6 +308,7 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
     cctx->jobIDMask = nbJobs - 1;
     cctx->allJobsCompleted = 1;
     cctx->sectionSize = 0;
+    cctx->overlapWrLog = 3;
     cctx->factory = POOL_create(nbThreads, 1);
     cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
     cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
@@ -367,6 +368,9 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter,
     case ZSTDMT_p_sectionSize :
         mtctx->sectionSize = value;
         return 0;
+    case ZSTDMT_p_overlapSectionRLog :
+        mtctx->overlapWrLog = value;
+        return 0;
     default :
         return ERROR(compressionParameter_unsupported);
     }
@@ -510,9 +514,7 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     zcs->frameContentSize = pledgedSrcSize;
     zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2);
     zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
-    //zcs->targetDictSize = ((size_t)1 << zcs->params.cParams.windowLog);   /* full window size, for test */
-    zcs->targetDictSize = ((size_t)1 << zcs->params.cParams.windowLog) >> 3;   /* fixed currently */
-    //zcs->targetDictSize = 0;
+    zcs->targetDictSize = zcs->overlapWrLog < 10 ? (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapWrLog) : 0;
     zcs->inBuffSize = zcs->targetSectionSize + ((size_t)1 << zcs->params.cParams.windowLog) /* margin */ + zcs->targetDictSize;
     zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
     if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation);

diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 4757e3e06d4d8432215d5c706984ae2723cee07b..92de52d652e1bcee59c3b8b49fc5bd03dd084ff2 100644 (file)
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -52,7 +52,8 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* d
 /* ZSDTMT_parameter :
  * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
 typedef enum {
-    ZSTDMT_p_sectionSize    /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
+    ZSTDMT_p_sectionSize,        /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
+    ZSTDMT_p_overlapSectionRLog  /* reverse log of overlapped section; 0 == use a complete window, 3(default) == use 1/8th of window, values >=10 means no overlap */
 } ZSDTMT_parameter;
 
 /* ZSTDMT_setMTCtxParameter() :

diff --git a/programs/fileio.c b/programs/fileio.c
index f18e418af0177b7d4cc2fdbc0e74718e9f4ec50e..ac7dffb31abdd772cfcc61155f189a7fcdbfd86a 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -7,6 +7,7 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
 
+
 /* *************************************
 *  Compiler Options
 ***************************************/
@@ -266,10 +267,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
 
 #ifdef ZSTD_MULTITHREAD
     ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
+    if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
+    if (cLevel==ZSTD_maxCLevel())
+        ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionRLog, 0);   /* use complete window for overlap */
 #else
     ress.cctx = ZSTD_createCStream();
-#endif
     if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
+#endif
     ress.srcBufferSize = ZSTD_CStreamInSize();
     ress.srcBuffer = malloc(ress.srcBufferSize);
     ress.dstBufferSize = ZSTD_CStreamOutSize();

diff --git a/programs/fileio.h b/programs/fileio.h
index 19f09c33a3239618d3f581c83a420a41d4f6c6d0..11178bcca94f862f36f42f127170c9bc8e9760ff 100644 (file)
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -12,12 +12,13 @@
 #define FILEIO_H_23981798732
 
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
-#include "zstd.h"     /* ZSTD_compressionParameters */
+#include "zstd.h"                  /* ZSTD_* */
 
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+
 /* *************************************
 *  Special i/o constants
 **************************************/

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 549dad01a2c97a2e8fd3837424005d7503706098..64f2c919c80a00024147d3ebb9626384417ae90e 100644 (file)
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -20,6 +20,7 @@
 #endif
 
 
+
 /*-************************************
 *  Dependencies
 **************************************/

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 2cb6d65e09785faa106c9c1ea04b3c355242d337..bef8734c7f866d2c2a9095e8e8223ee25300eebb 100644 (file)
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -992,8 +992,8 @@ int main(int argc, const char** argv)
     int mainPause = 0;
     int mtOnly = 0;
     const char* const programName = argv[0];
-    ZSTD_customMem customMem = { allocFunction, freeFunction, NULL };
-    ZSTD_customMem customNULL = { NULL, NULL, NULL };
+    ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL };
+    ZSTD_customMem const customNULL = { NULL, NULL, NULL };
 
     /* Check command line */
     for(argNb=1; argNb<argc; argNb++) {