From: Yann Collet Date: Tue, 19 May 2020 04:05:10 +0000 (-0700) Subject: fix 22294 (#2151) X-Git-Tag: v1.4.5^2~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=fdc56baa4248e903f364bca46362713531d24748;p=thirdparty%2Fzstd.git fix 22294 (#2151) --- diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index 9906e4ba2..85a3a7a91 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -398,7 +398,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; - DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic"); + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); /* switch to "regular" variant if extDict is invalidated due to maxDistance */ if (prefixStartIndex == dictStartIndex) @@ -415,6 +415,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; hashTable[h] = current; /* update hash table */ + DEBUGLOG(7, "offset_1 = %u , current = %u", offset_1, current); assert(offset_1 <= current +1); /* check repIndex */ if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 869986a0a..8c4794835 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -228,6 +228,7 @@ void ZSTD_ldm_fillHashTable( ldmState_t* state, const BYTE* ip, const BYTE* iend, ldmParams_t const* params) { + DEBUGLOG(5, "ZSTD_ldm_fillHashTable"); if ((size_t)(iend - ip) >= params->minMatchLength) { U64 startingHash = ZSTD_rollingHash_compute(ip, params->minMatchLength); ZSTD_ldm_fillLdmHashTable( @@ -594,7 +595,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, ZSTD_ldm_limitTableUpdate(ms, ip); ZSTD_ldm_fillFastTables(ms, ip); /* Run the block compressor */ - DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength); + DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); { size_t const newLitLength = blockCompressor(ms, seqStore, rep, ip, sequence.litLength); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 77a137bdf..5be7edfeb 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -461,9 +461,13 @@ typedef struct { ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ } serialState_t; -static int ZSTDMT_serialState_reset(serialState_t* serialState, - ZSTDMT_seqPool* seqPool, ZSTD_CCtx_params params, - size_t jobSize, const void* dict, size_t const dictSize) +static int +ZSTDMT_serialState_reset(serialState_t* serialState, + ZSTDMT_seqPool* seqPool, + ZSTD_CCtx_params params, + size_t jobSize, + const void* dict, size_t const dictSize, + ZSTD_dictContentType_e dictContentType) { /* Adjust parameters */ if (params.ldmParams.enableLdm) { @@ -510,10 +514,14 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, /* Update window state and fill hash table with dict */ if (dictSize > 0) { - BYTE const* const dictEnd = (const BYTE*)dict + dictSize; - ZSTD_window_update(&serialState->ldmState.window, dict, dictSize); - ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); - serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); + if (dictContentType == ZSTD_dct_rawContent) { + BYTE const* const dictEnd = (const BYTE*)dict + dictSize; + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize); + ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); + serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); + } else { + /* don't even load anything */ + } } /* Initialize serialState's copy of ldmWindow. */ @@ -1251,7 +1259,8 @@ ZSTDMT_computeNbJobs(const ZSTD_CCtx_params* params, size_t srcSize, unsigned nb /* ZSTDMT_compress_advanced_internal() : * This is a blocking function : it will only give back control to caller after finishing its compression job. */ -static size_t ZSTDMT_compress_advanced_internal( +static size_t +ZSTDMT_compress_advanced_internal( ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1283,7 +1292,8 @@ static size_t ZSTDMT_compress_advanced_internal( assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */ ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgJobSize) ); - if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize, NULL, 0)) + /* LDM doesn't even try to load the dictionary in single-ingestion mode */ + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize, NULL, 0, ZSTD_dct_auto)) return ERROR(memory_allocation); FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) , ""); /* only expands if necessary */ @@ -1516,7 +1526,8 @@ size_t ZSTDMT_initCStream_internal( mtctx->allJobsCompleted = 0; mtctx->consumed = 0; mtctx->produced = 0; - if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, dict, dictSize)) + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, + dict, dictSize, dictContentType)) return ERROR(memory_allocation); return 0; } diff --git a/tests/fuzz/.gitignore b/tests/fuzz/.gitignore index e2563097e..b6fc6e506 100644 --- a/tests/fuzz/.gitignore +++ b/tests/fuzz/.gitignore @@ -5,10 +5,16 @@ block_round_trip dictionary_decompress dictionary_loader dictionary_round_trip +dictionary_stream_round_trip +raw_dictionary_round_trip simple_compress simple_decompress simple_round_trip stream_decompress stream_round_trip zstd_frame_info -fuzz-*.log \ No newline at end of file +fuzz-*.log + +# misc +trace +tmp*