added ZSTD_compressBegin_targetSrcSize

diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c
index 2e62ec2c41a45a37f9bb31c90625e097e2dcf33a..43e6e663a4ede475c490cb8766ac9b51f3be0f82 100644 (file)
--- a/lib/zstd_compress.c
+++ b/lib/zstd_compress.c
@@ -97,6 +97,7 @@ struct ZSTD_CCtx_s
     U32   nextToUpdate;     /* index from which to continue dictionary update */
     U32   nextToUpdate3;    /* index from which to continue dictionary update */
     U32   hashLog3;         /* dispatch table : larger == faster, more memory */
+    U32   targetSrcSize;    /* optimize compression for this source size */
     U32   loadedDictEnd;
     U32   stage;
     ZSTD_parameters params;
@@ -240,6 +241,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     zc->hbSize = 0;
     zc->stage = 0;
     zc->loadedDictEnd = 0;
+    zc->targetSrcSize = 0;
 
     return 0;
 }
@@ -277,6 +279,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
     dstCCtx->dictLimit    = srcCCtx->dictLimit;
     dstCCtx->lowLimit     = srcCCtx->lowLimit;
     dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd;
+    dstCCtx->targetSrcSize= srcCCtx->targetSrcSize;
 
     /* copy entropy tables */
     dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
@@ -2180,6 +2183,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
     }
 }
 
+extern int g_additionalParam;
 
 /*! ZSTD_compressBegin_advanced() :
 *   @return : 0, or an error code */
@@ -2187,10 +2191,10 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
                              const void* dict, size_t dictSize,
                                    ZSTD_parameters params)
 {
-//    printf("windowLog=%d hashLog=%d\n", params.windowLog, params.hashLog);
+//    printf("windowLog=%d hashLog=%d targetSrcSize=%d\n", params.windowLog, params.hashLog, zc->targetSrcSize);
     ZSTD_validateParams(&params);
-    zc->hashLog3 = (params.searchLength==3) ? ZSTD_HASHLOG3 : 0;
-//    if (zc->hashLog3 > params.windowLog) zc->hashLog3 = params.windowLog;
+    U32 hashLog3 = (!zc->targetSrcSize || zc->targetSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((zc->targetSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);  
+    zc->hashLog3 = (params.searchLength==3) ? hashLog3 : 0;
 //    printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3);
 
     { size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params);
@@ -2220,6 +2224,15 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
 }
 
 
+size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel)
+{
+    zc->targetSrcSize = dictSize ? dictSize : targetSrcSize;
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, zc->targetSrcSize);
+    params.srcSize = 0;
+    ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_targetSrcSize compressionLevel=%d\n", zc->base, compressionLevel);
+    return ZSTD_compressBegin_advanced(zc, dict, dictSize, params);
+}
+
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, dictSize);
@@ -2309,12 +2322,14 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
 size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
+    ctx->targetSrcSize = srcSize;
     return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize));
 }
 
 size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel)
 {
     ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel);
+    ctx->targetSrcSize = srcSize;
     return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize));
 }
 

diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h
index 3b68654e371bb7288e7821fdf577c74db1fb44d4..1358eb8ab10bb5dbc03b53640f2f8168cd8eee24 100644 (file)
--- a/lib/zstd_internal.h
+++ b/lib/zstd_internal.h
@@ -218,6 +218,6 @@ typedef struct {
 } seqStore_t;
 
 seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx);
-
+size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel);
 
 #endif   /* ZSTD_CCOMMON_H_MODULE */

diff --git a/lib/zstd_static.h b/lib/zstd_static.h
index d41bc841ff02f6687901dc89b8137efd2b56041a..d88fbce825cd26f0e12acfec9f2fd4d849ab8f39 100644 (file)
--- a/lib/zstd_static.h
+++ b/lib/zstd_static.h
@@ -63,7 +63,8 @@ extern "C" {
 #define ZSTD_CONTENTLOG_MIN     4
 #define ZSTD_HASHLOG_MAX       28
 #define ZSTD_HASHLOG_MIN       12
-#define ZSTD_HASHLOG3          17
+#define ZSTD_HASHLOG3_MAX      17
+#define ZSTD_HASHLOG3_MIN      15
 #define ZSTD_SEARCHLOG_MAX    (ZSTD_CONTENTLOG_MAX-1)
 #define ZSTD_SEARCHLOG_MIN      1
 #define ZSTD_SEARCHLENGTH_MAX   7

diff --git a/programs/bench.c b/programs/bench.c
index d3e9c1f1df68c1030c035ed0e448481dde550829..1468bcb2b29705f8971469897c5ce4c1fbefb66b 100644 (file)
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -72,8 +72,9 @@
 
 #include "mem.h"
 #include "zstd_static.h"
+#include "zstd_internal.h" /* ZSTD_compressBegin_targetSrcSize */
+#include "datagen.h"       /* RDG_genBuffer */
 #include "xxhash.h"
-#include "datagen.h"      /* RDG_genBuffer */
 
 
 /* *************************************
@@ -139,9 +140,12 @@ static U32 g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result
 ***************************************/
 static U32 g_nbIterations = NBLOOPS;
 static size_t g_blockSize = 0;
+int g_additionalParam = 0;
 
 void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 
+void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
+
 void BMK_SetNbIterations(unsigned nbLoops)
 {
     g_nbIterations = nbLoops;
@@ -288,15 +292,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
 
             for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) {
                 U32 blockNb;
-#if 0
-                ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel);
-#else
-                ZSTD_parameters params = ZSTD_getParams(cLevel, dictBufferSize ? dictBufferSize : blockSize);
-           //     printf("cLevel=%d dictBufferSize=%d srcSize=%d params.srcSize=%d \n", cLevel, (int)dictBufferSize, (int)blockTable[0].srcSize, (int)params.srcSize);
-                params.srcSize = 0;
-                ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params);
-#endif
-
+                ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel);
                 for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                     size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
                                         blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
@@ -412,7 +408,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
 }
 
 static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
-                            const char* displayName, int cLevel, int cLevelLast, int additionalParam,
+                            const char* displayName, int cLevel, int cLevelLast,
                             const size_t* fileSizes, unsigned nbFiles,
                             const void* dictBuffer, size_t dictBufferSize)
 {
@@ -428,7 +424,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
     memset(&result, 0, sizeof(result));
     memset(&total, 0, sizeof(total));
 
-    if (g_displayLevel == 1 && !additionalParam)
+    if (g_displayLevel == 1 && !g_additionalParam)
         DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
 
     if (cLevelLast < cLevel) cLevelLast = cLevel;
@@ -439,8 +435,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
                      fileSizes, nbFiles,
                      dictBuffer, dictBufferSize, &result);
         if (g_displayLevel == 1) {
-            if (additionalParam)
-                DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s  %s (p=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, additionalParam);
+            if (g_additionalParam)
+                DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s  %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam);
             else
                 DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s  %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName);
             total.cSize += result.cSize;
@@ -491,7 +487,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
 }
 
 static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
-                               const char* dictFileName, int cLevel, int cLevelLast, int additionalParam)
+                               const char* dictFileName, int cLevel, int cLevelLast)
 {
     void* srcBuffer;
     size_t benchedSize;
@@ -531,7 +527,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
     else displayName = fileNamesTable[0];
 
     BMK_benchCLevel(srcBuffer, benchedSize,
-                    displayName, cLevel, cLevelLast, additionalParam,
+                    displayName, cLevel, cLevelLast,
                     fileSizes, nbFiles,
                     dictBuffer, dictBufferSize);
 
@@ -542,7 +538,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
 }
 
 
-static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, double compressibility)
+static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
 {
     char name[20] = {0};
     size_t benchedSize = 10000000;
@@ -556,7 +552,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d
 
     /* Bench */
     snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
-    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, additionalParam, &benchedSize, 1, NULL, 0);
+    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0);
 
     /* clean up */
     free(srcBuffer);
@@ -564,14 +560,14 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d
 
 
 int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
-                   const char* dictFileName, int cLevel, int cLevelLast, int additionalParam)
+                   const char* dictFileName, int cLevel, int cLevelLast)
 {
     double const compressibility = (double)g_compressibilityDefault / 100;
 
     if (nbFiles == 0)
-        BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility);
+        BMK_syntheticTest(cLevel, cLevelLast, compressibility);
     else
-        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, additionalParam);
+        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
     return 0;
 }
 

diff --git a/programs/bench.h b/programs/bench.h
index bc5ffa423e99522d3de6ae92868fcf4f44dc4f20..3a1ca3a2075243d46c3c8afef86a0cf600bc4342 100644 (file)
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -27,10 +27,11 @@
 
 /* Main function */
 int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
-                   const char* dictFileName, int cLevel, int cLevelLast, int additionalParam);
+                   const char* dictFileName, int cLevel, int cLevelLast);
 
 /* Set Parameters */
 void BMK_SetNbIterations(unsigned nbLoops);
 void BMK_SetBlockSize(size_t blockSize);
+void BMK_setAdditionalParam(int additionalParam);
 void BMK_setNotificationLevel(unsigned level);
 

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index a14fe26def13410c4119b0d8e21520830e4055f3..0c060bdbe5613dfd5c1e666138553c8976eb109c 100644 (file)
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -183,7 +183,6 @@ int main(int argCount, const char** argv)
         nextArgumentIsMaxDict=0;
     unsigned cLevel = 1;
     unsigned cLevelLast = 1;
-    int additionalParam = 0;
     const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));   /* argCount >= 1 */
     unsigned filenameIdx = 0;
     const char* programName = argv[0];
@@ -195,7 +194,7 @@ int main(int argCount, const char** argv)
     unsigned dictSelect = g_defaultSelectivityLevel;
 
     /* init */
-    (void)additionalParam; (void)cLevelLast; (void)dictCLevel;   /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */
+    (void)cLevelLast; (void)dictCLevel;   /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */
     if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
     displayOut = stderr;
     /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
@@ -321,7 +320,6 @@ int main(int argCount, const char** argv)
                             cLevelLast = 0;
                             while ((*argument >= '0') && (*argument <= '9'))
                                 cLevelLast *= 10, cLevelLast += *argument++ - '0';
-                            continue;
                         }
                         break;
 #endif   /* ZSTD_NOBENCH */
@@ -336,12 +334,14 @@ int main(int argCount, const char** argv)
                     /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
                 case 'p': argument++; 
                     if ((*argument>='0') && (*argument<='9')) {
-                        additionalParam = 0;
+                        int additionalParam = 0;
                         while ((*argument >= '0') && (*argument <= '9'))
                             additionalParam *= 10, additionalParam += *argument++ - '0';
-                        continue;
+                        BMK_setAdditionalParam(additionalParam);
+                    } else {
+                        main_pause=1;
                     }
-                    main_pause=1; break;
+                    break;
                     /* unknown command */
                 default : CLEAN_RETURN(badusage(programName));
                 }
@@ -383,7 +383,7 @@ int main(int argCount, const char** argv)
     if (bench) {
 #ifndef ZSTD_NOBENCH
         BMK_setNotificationLevel(displayLevel);
-        BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, additionalParam);
+        BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
 #endif
         goto _end;
     }