Fixed large NbSeq > 32 K

Added a test in Fuzzer to check NbSeq > 32 K

diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c
index a5127b74872e2720fab707e2a3736fc8917988a4..50979ce5838f13edaf0fcc7ac8bfb2f80d5b3c62 100644 (file)
--- a/lib/zstd_compress.c
+++ b/lib/zstd_compress.c
@@ -165,8 +165,8 @@ void ZSTD_validateParams(ZSTD_parameters* params)
         U32 srcLog = ZSTD_highbit((U32)(params->srcSize)-1) + 1;
         if (params->windowLog > srcLog) params->windowLog = srcLog;
     }
-    if (params->windowLog   < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* required for frame header */
-    if (params->contentLog  > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus;   /* <= ZSTD_CONTENTLOG_MAX */
+    if (params->windowLog  < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* required for frame header */
+    if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus;   /* <= ZSTD_CONTENTLOG_MAX */
 }
 
 
@@ -176,9 +176,12 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog);
     /* reserve table memory */
     const U32    contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
+    const U32    divider = (params.searchLength==3) ? 3 : 4;
+    const size_t maxNbSeq = blockSize / divider;
+    const size_t tokenSpace = blockSize + 8*maxNbSeq;
     const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog) + (1 << params.hashLog3)) * sizeof(U32);
     const size_t optSpace   = ((1<<MLbits) + (1<<LLbits) + (1<<Offbits) + (1<<Litbits))*sizeof(U32) + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
-    const size_t neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + (3*blockSize)
+    const size_t neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
                            + ((params.strategy == ZSTD_btopt) ? optSpace : 0);
 
     if (zc->workSpaceSize < neededSpace) {
@@ -206,13 +209,13 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     zc->blockSize = blockSize;
 
     zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
-    zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (blockSize>>2));
-    zc->seqStore.litStart = zc->seqStore.offCodeStart + (blockSize>>2);
+    zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + maxNbSeq);
+    zc->seqStore.litStart = zc->seqStore.offCodeStart + maxNbSeq;
     zc->seqStore.litLengthStart =  zc->seqStore.litStart + blockSize;
-    zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2);
-    zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2);
+    zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + maxNbSeq;
+    zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + maxNbSeq;
     if (params.strategy == ZSTD_btopt) {
-        zc->seqStore.litFreq = (U32*)((void*)(zc->seqStore.dumpsStart + (blockSize>>2)));
+        zc->seqStore.litFreq = (U32*)((void*)(zc->seqStore.dumpsStart + maxNbSeq));
         zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
         zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<<LLbits);
         zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<<MLbits);

diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 6d57080d3f6ca7128939bcb29e81457d5c92cf20..697695b382b70e5d5e3c067524172070ecd1ba8c 100644 (file)
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -344,7 +344,6 @@ static int basicUnitTests(U32 seed, double compressibility)
     DISPLAYLEVEL(4, "OK \n");
 
     /* nbSeq limit test */
-    if (0)
     {
         #define _3BYTESTESTLENGTH 131000
         #define NB3BYTESSEQLOG   9
@@ -360,7 +359,7 @@ static int basicUnitTests(U32 seed, double compressibility)
             _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&r) & 255);
         }
 
-        for (i=0; i < _3BYTESTESTLENGTH; ){
+        for (i=0; i < _3BYTESTESTLENGTH; ) {
             U32 id = FUZ_rand(&r) & NB3BYTESSEQMASK;
             ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
             ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];