From: Yann Collet Date: Fri, 29 Dec 2017 16:04:37 +0000 (+0100) Subject: fixed bug in dubt X-Git-Tag: v1.3.4~1^2~87^2~14 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=64482c2c9756cbe4f0003ee92fa0955aead7a6b8;p=thirdparty%2Fzstd.git fixed bug in dubt the chain of unsorted candidates could grow beyond lowLimit. --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 91e281660..01d1b5d74 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -752,9 +752,9 @@ static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t blockSize1, size_t const windowSize2 = MAX(1, (size_t)MIN(((U64)1 << cParams2.windowLog), pledgedSrcSize)); size_t const blockSize2 = MIN(ZSTD_BLOCKSIZE_MAX, windowSize2); size_t const neededBufferSize2 = (buffPol2==ZSTDb_buffered) ? windowSize2 + blockSize2 : 0; - DEBUGLOG(4, "ZSTD_sufficientBuff: windowSize2=%u from wlog=%u", + DEBUGLOG(4, "ZSTD_sufficientBuff: is windowSize2=%u <= wlog1=%u", (U32)windowSize2, cParams2.windowLog); - DEBUGLOG(4, "ZSTD_sufficientBuff: blockSize2 %u <=? blockSize1 %u", + DEBUGLOG(4, "ZSTD_sufficientBuff: is blockSize2=%u <= blockSize1=%u", (U32)blockSize2, (U32)blockSize1); return (blockSize2 <= blockSize1) /* seqStore space depends on blockSize */ & (neededBufferSize2 <= bufferSize1); @@ -777,10 +777,12 @@ static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, * reuse CCtx without reset (note : requires no dictionary) */ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize) { - U32 const end = (U32)(cctx->nextSrc - cctx->base); + size_t const endT = (size_t)(cctx->nextSrc - cctx->base); + U32 const end = (U32)endT; size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - DEBUGLOG(4, "ZSTD_continueCCtx"); + DEBUGLOG(4, "ZSTD_continueCCtx: re-use context in place"); + assert(endT < (3U<<30)); cctx->blockSize = blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */ cctx->appliedParams = params; @@ -912,12 +914,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->entropy->offcode_repeatMode = FSE_repeat_none; zc->entropy->matchlength_repeatMode = FSE_repeat_none; zc->entropy->litlength_repeatMode = FSE_repeat_none; - zc->nextToUpdate = 1; - zc->nextSrc = NULL; zc->base = NULL; zc->dictBase = NULL; - zc->dictLimit = 0; zc->lowLimit = 0; + zc->dictLimit = 0; + zc->nextToUpdate = 1; + zc->nextSrc = NULL; { int i; for (i=0; iseqStore.rep[i] = repStartValue[i]; } zc->hashLog3 = hashLog3; zc->optState.litLengthSum = 0; @@ -925,7 +927,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ptr = zc->entropy + 1; /* opt parser space */ - if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) { + if ((params.cParams.strategy == ZSTD_btopt) | (params.cParams.strategy == ZSTD_btultra)) { DEBUGLOG(4, "reserving optimal parser space"); assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ zc->optState.litFreq = (U32*)ptr; @@ -1643,9 +1645,12 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->longLengthID = 0; } -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { - DEBUGLOG(5, "ZSTD_compressBlock_internal : dstCapacity = %u", (U32)dstCapacity); + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u) (dictLimit=%u, nextToUpdate=%u)", + (U32)dstCapacity, zc->dictLimit, zc->nextToUpdate); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ ZSTD_resetSeqStore(&(zc->seqStore)); @@ -1737,13 +1742,17 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, cctx->dictLimit -= correction; if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0; else cctx->nextToUpdate -= correction; - DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x\n", correction, cctx->lowLimit); + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, cctx->lowLimit); } /* enforce maxDist */ if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; + if (cctx->dictLimit < cctx->lowLimit) + DEBUGLOG(2, "ZSTD_compress_frameChunk : update dictLimit from %u to %u ", + cctx->dictLimit, cctx->lowLimit); if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; + if (cctx->nextToUpdate < cctx->lowLimit) cctx->nextToUpdate = cctx->lowLimit; } { size_t cSize = ZSTD_compressBlock_internal(cctx, @@ -1846,8 +1855,9 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, /* Check if blocks follow each other */ if (src != cctx->nextSrc) { - /* not contiguous */ size_t const distanceFromBase = (size_t)(cctx->nextSrc - cctx->base); + DEBUGLOG(5, "ZSTD_compressContinue_internal: non contiguous blocks, new segment starts at %u", + cctx->dictLimit); cctx->lowLimit = cctx->dictLimit; assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ cctx->dictLimit = (U32)distanceFromBase; @@ -1879,6 +1889,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { + DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (U32)srcSize); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); } @@ -2144,10 +2155,10 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); - DEBUGLOG(4, "ZSTD_compressBegin_usingDict"); + DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (U32)dictSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); } @@ -2275,6 +2286,7 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, co size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { + DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (U32)srcSize); return ZSTD_compress_usingDict(ctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); } @@ -2885,7 +2897,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, if (params.nbThreads > 1) { if (cctx->mtctx == NULL || (params.nbThreads != ZSTDMT_getNbThreads(cctx->mtctx))) { DEBUGLOG(4, "ZSTD_compress_generic: creating new mtctx for nbThreads=%u (previous: %u)", - params.nbThreads, ZSTDMT_getNbThreads(cctx->mtctx)); + params.nbThreads, (unsigned)ZSTDMT_getNbThreads(cctx->mtctx)); ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbThreads, cctx->customMem); if (cctx->mtctx == NULL) return ERROR(memory_allocation); diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 431157741..5249c5e58 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -32,12 +32,12 @@ void ZSTD_updateDUBT(ZSTD_CCtx* zc, U32 const target = (U32)(ip - base); U32 idx = zc->nextToUpdate; - DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u ", - idx, target); + if (idx != target) + DEBUGLOG(2, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", + idx, target, zc->dictLimit); assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ (void)iend; - assert(idx >= zc->dictLimit); /* condition for valid base+idx */ for ( ; idx < target ; idx++) { size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ @@ -79,10 +79,11 @@ static void ZSTD_insertDUBT1(ZSTD_CCtx* zc, U32 dummy32; /* to be nullified at the end */ U32 const windowLow = zc->lowLimit; - DEBUGLOG(8, "ZSTD_insertDUBT1 (%u)", current); + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", + current, dictLimit, windowLow); assert(current >= btLow); - if (extDict && (current < dictLimit)) { /* do not sort candidates in _extDict (simplification, for easier ZSTD_count, detrimental to compression ratio in streaming mode) */ + if (extDict && (current < dictLimit)) { /* candidates in _extDict are not sorted (simplification, for easier ZSTD_count, detrimental to compression ratio in streaming mode) */ *largerPtr = *smallerPtr = 0; return; } @@ -147,11 +148,13 @@ static size_t ZSTD_insertBtAndFindBestMatch ( const BYTE* const base = zc->base; U32 const current = (U32)(ip-base); + U32 const windowLow = zc->lowLimit; U32* const bt = zc->chainTable; U32 const btLog = zc->appliedParams.cParams.chainLog - 1; U32 const btMask = (1 << btLog) - 1; U32 const btLow = (btMask >= current) ? 0 : current - btMask; + U32 const unsortLimit = MAX(btLow, windowLow); U32* nextCandidate = bt + 2*(matchIndex&btMask); U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1; @@ -162,7 +165,7 @@ static size_t ZSTD_insertBtAndFindBestMatch ( assert(ip <= iend-8); /* required for h calculation */ /* reach end of unsorted candidates list */ - while ( (matchIndex > btLow) + while ( (matchIndex > unsortLimit) && (*unsortedMark == ZSTD_DUBT_UNSORTED) && (nbCandidates > 1) ) { DEBUGLOG(8, "ZSTD_insertBtAndFindBestMatch: candidate %u is unsorted", @@ -175,11 +178,11 @@ static size_t ZSTD_insertBtAndFindBestMatch ( nbCandidates --; } - if ( (matchIndex > btLow) + if ( (matchIndex > unsortLimit) && (*unsortedMark==ZSTD_DUBT_UNSORTED) ) { DEBUGLOG(8, "ZSTD_insertBtAndFindBestMatch: nullify last unsorted candidate %u", matchIndex); - *nextCandidate = *unsortedMark = 0; /* nullify last candidate if it's still unsorted (note : detrimental to compression ratio) */ + *nextCandidate = *unsortedMark = 0; /* nullify next candidate if it's still unsorted (note : simplification, detrimental to compression ratio, beneficial for speed) */ } /* batch sort stacked candidates */ @@ -188,7 +191,7 @@ static size_t ZSTD_insertBtAndFindBestMatch ( U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; U32 const nextCandidateIdx = *nextCandidateIdxPtr; ZSTD_insertDUBT1(zc, matchIndex, iend, - nbCandidates, btLow, extDict); + nbCandidates, unsortLimit, extDict); matchIndex = nextCandidateIdx; nbCandidates++; } @@ -199,7 +202,6 @@ static size_t ZSTD_insertBtAndFindBestMatch ( const U32 dictLimit = zc->dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - const U32 windowLow = zc->lowLimit; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; U32 matchEndIdx = current+8+1; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 141bf6548..3e911d7ca 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -286,91 +286,91 @@ static int basicUnitTests(U32 seed, double compressibility) RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); /* Basic tests */ - DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName : ", testNb++); + DISPLAYLEVEL(3, "test%3i : ZSTD_getErrorName : ", testNb++); { const char* errorString = ZSTD_getErrorName(0); - DISPLAYLEVEL(4, "OK : %s \n", errorString); + DISPLAYLEVEL(3, "OK : %s \n", errorString); } - DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++); + DISPLAYLEVEL(3, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++); { const char* errorString = ZSTD_getErrorName(499); - DISPLAYLEVEL(4, "OK : %s \n", errorString); + DISPLAYLEVEL(3, "OK : %s \n", errorString); } - DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize); + DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize); { ZSTD_CCtx* cctx = ZSTD_createCCtx(); if (cctx==NULL) goto _output_error; CHECKPLUS(r, ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, 1), cSize=r ); - DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); - DISPLAYLEVEL(4, "test%3i : size of cctx for level 1 : ", testNb++); + DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++); { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); - DISPLAYLEVEL(4, "%u bytes \n", (U32)cctxSize); + DISPLAYLEVEL(3, "%u bytes \n", (U32)cctxSize); } ZSTD_freeCCtx(cctx); } - DISPLAYLEVEL(4, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); + DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); if (rSize != CNBuffSize) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); + DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize); if (rSize != CNBuffSize) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); + DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); if (r != CNBuffSize) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); { size_t u; for (u=0; u (1U << 20)) goto _output_error; ZSTD_freeCCtx(cctx); } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%di : re-use CCtx with expanding block size : ", testNb++); + DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */ @@ -385,11 +385,11 @@ static int basicUnitTests(U32 seed, double compressibility) } ZSTD_freeCCtx(cctx); } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); /* Static CCtx tests */ #define STATIC_CCTX_LEVEL 3 - DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL); + DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL); { size_t const staticCCtxSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL); void* const staticCCtxBuffer = malloc(staticCCtxSize); size_t const staticDCtxSize = ZSTD_estimateDCtxSize(); @@ -404,57 +404,57 @@ static int basicUnitTests(U32 seed, double compressibility) { ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); ZSTD_DCtx* staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize); if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error; - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); + DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); { size_t const r = ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL); if (ZSTD_isError(r)) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : simple compression test with static CCtx : ", testNb++); + DISPLAYLEVEL(3, "test%3i : simple compression test with static CCtx : ", testNb++); CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL), cSize=r ); - DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); - DISPLAYLEVEL(4, "test%3i : simple decompression test with static DCtx : ", testNb++); + DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++); { size_t const r = ZSTD_decompressDCtx(staticDCtx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); if (r != CNBuffSize) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); { size_t u; for (u=0; u same size */ } - DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); - DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); + DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, CNBuffer, dictSize), if (r != CNBuffSize - dictSize) goto _output_error); - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : decompress with DDict : ", testNb++); + DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++); { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize); size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); if (r != CNBuffSize - dictSize) goto _output_error; - DISPLAYLEVEL(4, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict)); + DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict)); ZSTD_freeDDict(ddict); } - DISPLAYLEVEL(4, "test%3i : decompress with static DDict : ", testNb++); + DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++); { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); void* ddictBuffer = malloc(ddictBufferSize); if (ddictBuffer == NULL) goto _output_error; @@ -633,10 +632,10 @@ static int basicUnitTests(U32 seed, double compressibility) if (r != CNBuffSize - dictSize) goto _output_error; } free(ddictBuffer); - DISPLAYLEVEL(4, "OK (size of static DDict : %u) \n", (U32)ddictBufferSize); + DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (U32)ddictBufferSize); } - DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++); + DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); { size_t const testSize = CNBuffSize / 3; { ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize); p.fParams.contentSizeFlag = 1; @@ -651,7 +650,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error; if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error; } } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); ZSTD_freeCCtx(ctxOrig); ZSTD_freeCCtx(ctxDuplicated); @@ -673,79 +672,79 @@ static int basicUnitTests(U32 seed, double compressibility) goto _output_error; } - DISPLAYLEVEL(4, "test%3i : dictBuilder : ", testNb++); + DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++); { U32 u; for (u=0; u %u bytes)\n", (U32)inputSize, (U32)cSize); + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize); - DISPLAYLEVEL(4, "test%3i : decompress normally (should fail) : ", testNb++); + DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++); { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error; - DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(decodeResult)); + DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult)); } - DISPLAYLEVEL(4, "test%3i : decompress with magic-less instruction : ", testNb++); + DISPLAYLEVEL(3, "test%3i : decompress with magic-less instruction : ", testNb++); ZSTD_DCtx_reset(dctx); CHECK( ZSTD_DCtx_setFormat(dctx, ZSTD_f_zstd1_magicless) ); { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; @@ -1019,7 +1018,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (result != 0) goto _output_error; if (in.pos != in.size) goto _output_error; if (out.pos != inputSize) goto _output_error; - DISPLAYLEVEL(4, "OK : regenerated %u bytes \n", (U32)out.pos); + DISPLAYLEVEL(3, "OK : regenerated %u bytes \n", (U32)out.pos); } ZSTD_freeCCtx(cctx); @@ -1032,21 +1031,21 @@ static int basicUnitTests(U32 seed, double compressibility) size_t cSize2; /* basic block compression */ - DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); + DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); CHECK( ZSTD_compressBegin(cctx, 5) ); CHECK( ZSTD_getBlockSize(cctx) >= blockSize); cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); if (ZSTD_isError(cSize)) goto _output_error; - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); + DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++); CHECK( ZSTD_decompressBegin(dctx) ); { CHECK_V(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); if (r != blockSize) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); /* dictionary block compression */ - DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++); + DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize); if (ZSTD_isError(cSize)) goto _output_error; @@ -1056,16 +1055,16 @@ static int basicUnitTests(U32 seed, double compressibility) cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+2*blockSize, blockSize); if (ZSTD_isError(cSize2)) goto _output_error; - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); + DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++); CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); { CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); if (r != blockSize) goto _output_error; } ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */ { CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, CNBuffSize, (char*)compressedBuffer+cSize+blockSize, cSize2) ); if (r != blockSize) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); ZSTD_freeCCtx(cctx); } @@ -1073,7 +1072,7 @@ static int basicUnitTests(U32 seed, double compressibility) /* long rle test */ { size_t sampleSize = 0; - DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++); + DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); sampleSize += 256 KB - 1; @@ -1083,21 +1082,21 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_isError(cSize)) goto _output_error; { CHECK_V(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); if (regenSize!=sampleSize) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); } /* All zeroes test (test bug #137) */ #define ZEROESLENGTH 100 - DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); + DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); memset(CNBuffer, 0, ZEROESLENGTH); { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); cSize = r; } - DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100); - DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); + DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); { CHECK_V(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); if (r != ZEROESLENGTH) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); /* nbSeq limit test */ #define _3BYTESTESTLENGTH 131000 @@ -1124,18 +1123,18 @@ static int basicUnitTests(U32 seed, double compressibility) ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; } } } - DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++); + DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++); { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19) ); cSize = r; } - DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100); - DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++); + DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++); { CHECK_V(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); if (r != _3BYTESTESTLENGTH) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : incompressible data and ill suited dictionary : ", testNb++); + DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++); RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed); { /* Train a dictionary on low characters */ size_t dictSize = 16 KB; @@ -1168,25 +1167,25 @@ static int basicUnitTests(U32 seed, double compressibility) free(dictBuffer); free(samplesSizes); } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); /* findFrameCompressedSize on skippable frames */ - DISPLAYLEVEL(4, "test%3i : frame compressed size of skippable frame : ", testNb++); + DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++); { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde"; size_t const frameSrcSize = 13; if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; } - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); /* error string tests */ - DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++); + DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++); if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error; if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error; if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error; if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error; if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error; if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error; - DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); _end: free(CNBuffer); @@ -1369,6 +1368,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD } /* successful decompression test */ + DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb); { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); @@ -1379,6 +1379,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD free(sampleBuffer); /* no longer useful after this point */ /* truncated src decompression test */ + DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb); { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ size_t const tooSmallSize = cSize - missing; void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ @@ -1390,6 +1391,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD } /* too small dst decompression test */ + DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb); if (sampleSize > 3) { size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ size_t const tooSmallSize = sampleSize - missing; @@ -1425,6 +1427,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD } } } /* decompress noisy source */ + DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb); { U32 const endMark = 0xA9B1C3D6; memcpy(dstBuffer+sampleSize, &endMark, 4); { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); @@ -1436,8 +1439,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); } } } /* noisy src decompression test */ - /*===== Streaming compression test, scattered segments and dictionary =====*/ - + /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/ + DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb); { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; int const cLevel = (FUZ_rand(&lseed) % @@ -1450,10 +1453,13 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */ dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); + DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n", + testNb, (U32)maxTestSize, cLevel, (U32)dictSize); + if (FUZ_rand(&lseed) & 0xF) { CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) ); } else { - ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize); + ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ @@ -1491,6 +1497,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD } /* streaming decompression test */ + DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb); /* ensure memory requirement is good enough (should always be true) */ { ZSTD_frameHeader zfh; CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_frameHeaderSize_max), @@ -1633,7 +1640,7 @@ int main(int argc, const char** argv) case 'v': argument++; - g_displayLevel = 4; + g_displayLevel++; break; case 'q':