From: Yann Collet Date: Mon, 26 Sep 2016 14:41:05 +0000 (+0200) Subject: fixed : init*_advanced() followed by reset() with different pledgedSrcSiz X-Git-Tag: v1.1.0~5 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=cf409a7e2a4182530496f1b690a6c0a898b6d8dc;p=thirdparty%2Fzstd.git fixed : init*_advanced() followed by reset() with different pledgedSrcSiz --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 298278c99..94f4b5a25 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -142,21 +142,8 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) } -/** ZSTD_checkCParams_advanced() : - temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */ -size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize) -{ - if (srcSize > (1ULL << ZSTD_WINDOWLOG_MIN)) return ZSTD_checkCParams(cParams); - if (cParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) return ERROR(compressionParameter_unsupported); - if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */ - if (srcSize <= (1ULL << cParams.chainLog)) cParams.chainLog = ZSTD_CHAINLOG_MIN; /* fake value - temporary work around */ - if ((srcSize <= (1ULL << cParams.hashLog)) & ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */ - return ZSTD_checkCParams(cParams); -} - - /** ZSTD_adjustCParams() : - optimize cPar for a given input (`srcSize` and `dictSize`). + optimize `cPar` for a given input (`srcSize` and `dictSize`). mostly downsizing to reduce memory consumption and initialization. Both `srcSize` and `dictSize` are optional (use 0 if unknown), but if both are 0, no optimization can be done. @@ -169,7 +156,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1< srcLog) cPar.windowLog = srcLog; } } if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; @@ -178,7 +165,6 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ - if ((cPar.hashLog < ZSTD_HASHLOG_MIN) & ((U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ return cPar; } @@ -2556,7 +2542,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, ZSTD_parameters params, unsigned long long pledgedSrcSize) { /* compression parameters verification and optimization */ - CHECK_F(ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize)); + CHECK_F(ZSTD_checkCParams(params.cParams)); return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); } @@ -2644,7 +2630,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params) { - CHECK_F(ZSTD_checkCParams_advanced(params.cParams, srcSize)); + CHECK_F(ZSTD_checkCParams(params.cParams)); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); } @@ -2851,7 +2837,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; if (zcs->inBuffSize < neededInBuffSize) { zcs->inBuffSize = neededInBuffSize; - ZSTD_free(zcs->inBuff, zcs->customMem); /* should not be necessary */ + ZSTD_free(zcs->inBuff, zcs->customMem); zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem); if (zcs->inBuff == NULL) return ERROR(memory_allocation); } @@ -2859,7 +2845,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, } if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; - ZSTD_free(zcs->outBuff, zcs->customMem); /* should not be necessary */ + ZSTD_free(zcs->outBuff, zcs->customMem); zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem); if (zcs->outBuff == NULL) return ERROR(memory_allocation); } diff --git a/lib/zstd.h b/lib/zstd.h index d7eb9c01f..dd3f5df4c 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -290,11 +290,11 @@ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* outp #define ZSTD_WINDOWLOG_MAX_32 25 #define ZSTD_WINDOWLOG_MAX_64 27 #define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) -#define ZSTD_WINDOWLOG_MIN 18 -#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) -#define ZSTD_CHAINLOG_MIN 4 +#define ZSTD_WINDOWLOG_MIN 10 #define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX -#define ZSTD_HASHLOG_MIN 12 +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN #define ZSTD_HASHLOG3_MAX 17 #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) #define ZSTD_SEARCHLOG_MIN 1 diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 7dcd8ea07..8486013c2 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -374,7 +374,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres ZSTD_DStream* const zd_noise = ZSTD_createDStream(); clock_t const startClock = clock(); const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ - size_t dictSize=0, maxTestSize=0; + size_t dictSize = 0; + U32 oldTestLog = 0; /* allocations */ cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); @@ -407,6 +408,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres XXH64_state_t xxhState; U64 crcOrig; U32 resetAllowed = 1; + size_t maxTestSize; /* init */ DISPLAYUPDATE(2, "\r%6u", testNb); @@ -435,23 +437,29 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres } /* compression init */ - if (maxTestSize /* at least one test happened */ && resetAllowed && (FUZ_rand(&lseed)&1)) { - U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; - ZSTD_resetCStream(zc, pledgedSrcSize); + if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ + && oldTestLog /* at least one test happened */ && resetAllowed) { + maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); + if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1; + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; + size_t const resetError = ZSTD_resetCStream(zc, pledgedSrcSize); + CHECK(ZSTD_isError(resetError), "ZSTD_resetCStream error : %s", ZSTD_getErrorName(resetError)); + } } else { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; maxTestSize = FUZ_rLogLength(&lseed, testLog); + oldTestLog = testLog; /* random dictionary selection */ dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); dict = srcBuffer + dictStart; } - { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize); + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; + ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; - size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize); + { size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize); CHECK (ZSTD_isError(initError),"ZSTD_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); } } }