ensure adjustCParams adjust hLog and cLog even without srcSize

It would previously exit when srcSize is unknown.
But in the case of custom parameters,
hLog and cLog can still be too large in comparison with windowLog.

Reduces maximum memory allocated during zstreamtest --newapi

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e84149bd310aa1e4f198a8e2e2c98f9698659e64..0dbe196ff9c149d1145aef8e465d26c8adbf3761 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -634,26 +634,31 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
     mostly downsizing to reduce memory consumption and initialization.
     Both `srcSize` and `dictSize` are optional (use 0 if unknown),
     but if both are 0, no optimization can be done.
-    Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
+    Note : cPar is considered validated at this stage. Use ZSTD_checkCParams() to ensure that condition. */
 ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
 {
     assert(ZSTD_checkCParams(cPar)==0);
-    if (srcSize+dictSize == 0) return cPar;   /* no size information available : no adjustment */
 
-    /* resize params, to use less memory when necessary */
-    {   U32 const minSrcSize = (srcSize==0) ? 500 : 0;
+    /* resize windowLog if src is small, to use less memory when necessary */
+    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == -1ULL);
+    if ( (dictSize || (srcSize+1 > 1))  /* srcSize test depends on static assert condition */
+      && (srcSize-1 < (1ULL<<ZSTD_WINDOWLOG_MAX)) ) /* no correction is srcSize is large enough */ {
+        U32 const minSrcSize = (srcSize==0) ? 513 : 0;
         U64 const rSize = srcSize + dictSize + minSrcSize;
-        if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
+        if (rSize < (1ULL<<ZSTD_WINDOWLOG_MAX)) {
             U32 const srcLog =
-                    MAX(ZSTD_HASHLOG_MIN, (rSize==1) ? 1 : ZSTD_highbit32((U32)(rSize)-1) + 1);
+                    MAX(ZSTD_HASHLOG_MIN,
+                        (rSize==1) ? 1 : ZSTD_highbit32((U32)(rSize)-1) + 1);
             if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
     }   }
     if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
     {   U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
-        if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog);
+        if (cycleLog > cPar.windowLog)
+            cPar.chainLog -= (cycleLog - cPar.windowLog);
     }
 
-    if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* required for frame header */
+    if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
+        cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* required for frame header */
 
     return cPar;
 }

diff --git a/lib/zstd.h b/lib/zstd.h
index b7c75d5f250f63af6852324c16d9692dba4c40f1..59aa0b020043bcb44980c254cbfad4a9b1b37abf 100644 (file)
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -376,22 +376,22 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
 #define ZSTD_MAGIC_DICTIONARY       0xEC30A437   /* v0.7+ */
 
-#define ZSTD_WINDOWLOG_MAX_32  30
-#define ZSTD_WINDOWLOG_MAX_64  31
+#define ZSTD_WINDOWLOG_MAX_32   30
+#define ZSTD_WINDOWLOG_MAX_64   31
 #define ZSTD_WINDOWLOG_MAX    ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
-#define ZSTD_WINDOWLOG_MIN     10
-#define ZSTD_HASHLOG_MAX       MIN(ZSTD_WINDOWLOG_MAX, 30)
-#define ZSTD_HASHLOG_MIN        6
-#define ZSTD_CHAINLOG_MAX      MIN(ZSTD_WINDOWLOG_MAX+1, 30)
-#define ZSTD_CHAINLOG_MIN      ZSTD_HASHLOG_MIN
-#define ZSTD_HASHLOG3_MAX      17
-#define ZSTD_SEARCHLOG_MAX    (ZSTD_WINDOWLOG_MAX-1)
-#define ZSTD_SEARCHLOG_MIN      1
-#define ZSTD_SEARCHLENGTH_MAX   7   /* only for ZSTD_fast, other strategies are limited to 6 */
-#define ZSTD_SEARCHLENGTH_MIN   3   /* only for ZSTD_btopt, other strategies are limited to 4 */
-#define ZSTD_TARGETLENGTH_MIN   4
-#define ZSTD_TARGETLENGTH_MAX 999
-#define ZSTD_LDM_MINMATCH_MIN 4
+#define ZSTD_WINDOWLOG_MIN      10
+#define ZSTD_HASHLOG_MAX        MIN(ZSTD_WINDOWLOG_MAX, 30)
+#define ZSTD_HASHLOG_MIN         6
+#define ZSTD_CHAINLOG_MAX       MIN(ZSTD_WINDOWLOG_MAX+1, 30)
+#define ZSTD_CHAINLOG_MIN       ZSTD_HASHLOG_MIN
+#define ZSTD_HASHLOG3_MAX       17
+#define ZSTD_SEARCHLOG_MAX     (ZSTD_WINDOWLOG_MAX-1)
+#define ZSTD_SEARCHLOG_MIN       1
+#define ZSTD_SEARCHLENGTH_MAX    7   /* only for ZSTD_fast, other strategies are limited to 6 */
+#define ZSTD_SEARCHLENGTH_MIN    3   /* only for ZSTD_btopt, other strategies are limited to 4 */
+#define ZSTD_TARGETLENGTH_MIN    4   /* only useful for btopt */
+#define ZSTD_TARGETLENGTH_MAX  999   /* only useful for btopt */
+#define ZSTD_LDM_MINMATCH_MIN    4
 #define ZSTD_LDM_MINMATCH_MAX 4096
 #define ZSTD_LDM_BUCKETSIZELOG_MAX 8
 

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 62af8e167a59b6d6e55eeaf4f5eb1539399988c8..0b8995c5d7bd4abc4ebc15673b4f792697b19a8e 100644 (file)
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1322,6 +1322,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
         else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
         FUZ_rand(&coreSeed);
         lseed = coreSeed ^ prime32;
+        DISPLAYLEVEL(5, " ***  Test %u  *** \n", testNb);
 
         /* states full reset (deliberately not synchronized) */
         /* some issues can only happen when reusing states */
@@ -1371,7 +1372,9 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                                (MAX(testLog, dictLog) / 3))) +
                                1;
             U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
+            DISPLAYLEVEL(5, "t%u: cLevel : %u \n", testNb, cLevel);
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
+            DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize);
             oldTestLog = testLog;
             /* random dictionary selection */
             dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
@@ -1396,9 +1399,16 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                     CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) );
                     assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN);   /* guaranteed by ZSTD_adjustCParams() */
                     windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
+                    DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
+                }
+                if (FUZ_rand(&lseed) & 1) {
+                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
+                    DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
+                }
+                if (FUZ_rand(&lseed) & 1) {
+                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
+                    DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
                 }
-                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) );
@@ -1415,13 +1425,14 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
-                DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
+                DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
 
                 /* multi-threading parameters */
                 {   U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
                     U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
                     U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
                     CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
+                    DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
                     if (nbThreads > 1) {
                         U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
                         CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) );