From: Yann Collet Date: Mon, 5 Aug 2019 13:18:43 +0000 (+0200) Subject: factored the logic selecting lowest match index X-Git-Tag: v1.4.3^2~5^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=facbe8b2c250a95a9720bb47af89d481dc5939fb;p=thirdparty%2Fzstd.git factored the logic selecting lowest match index as suggested by @terrelln --- diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 9a511e55c..6d623cc6b 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -136,6 +136,8 @@ struct ZSTD_matchState_t { ZSTD_window_t window; /* State for window round buffer management */ U32 loadedDictEnd; /* index of end of dictionary, within context's referential. * When loadedDictEnd != 0, a dictionary is in use, and still valid. + * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. + * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). * When dict referential is copied into active context (i.e. not attached), * loadedDictEnd == dictSize, since referential starts from zero. */ @@ -354,7 +356,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - ZSTD_wildcopy(seqStorePtr->lit, literals, litLength, ZSTD_no_overlap); + ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap); seqStorePtr->lit += litLength; /* literal Length */ @@ -839,6 +841,17 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, return contiguous; } +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + + /* debug functions */ #if (DEBUGLEVEL>=2) diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c index f1c6520e1..54467cc31 100644 --- a/lib/compress/zstd_double_fast.c +++ b/lib/compress/zstd_double_fast.c @@ -65,6 +65,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); const U32 lowestValid = ms->window.dictLimit; const U32 maxDistance = 1U << cParams->windowLog; + /* presumes that, if there is a dictionary, it must be using Attach mode */ const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; const BYTE* const prefixLowest = base + prefixLowestIndex; const BYTE* const iend = istart + srcSize; @@ -369,11 +370,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const ilimit = iend - 8; const BYTE* const base = ms->window.base; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 maxDistance = 1U << cParams->windowLog; - const U32 validLowest = ms->window.lowLimit; - const int isDictionary = (ms->loadedDictEnd != 0); - const U32 withinWindow = (endIndex - validLowest > maxDistance) ? endIndex - maxDistance : validLowest; - const U32 lowLimit = isDictionary ? validLowest : withinWindow; + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); const U32 dictStartIndex = lowLimit; const U32 dictLimit = ms->window.dictLimit; const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index fd7869f5c..59267ffbb 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -381,11 +381,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* ip = istart; const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 maxDistance = 1U << cParams->windowLog; - const U32 validLowest = ms->window.lowLimit; - const int isDictionary = (ms->loadedDictEnd != 0); - const U32 withinWindow = (endIndex - validLowest > maxDistance) ? endIndex - maxDistance : validLowest; - const U32 lowLimit = isDictionary ? validLowest : withinWindow; + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); const U32 dictStartIndex = lowLimit; const BYTE* const dictStart = dictBase + dictStartIndex; const U32 dictLimit = ms->window.dictLimit; diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 4d7cf5508..0af41724c 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -242,11 +242,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); - U32 const maxDistance = 1U << cParams->windowLog; - U32 const lowestValid = ms->window.lowLimit; - U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; - U32 const isDictionary = (ms->loadedDictEnd != 0); - U32 const windowLow = isDictionary ? lowestValid : withinWindow; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog); U32* const bt = ms->chainTable; U32 const btLog = cParams->chainLog - 1; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index b12b14629..2da363f93 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -552,7 +552,6 @@ U32 ZSTD_insertBtAndGetAllMatches ( { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); - U32 const maxDistance = 1U << cParams->windowLog; const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); U32 const hashLog = cParams->hashLog; @@ -569,10 +568,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; U32 const btLow = (btMask >= current) ? 0 : current - btMask; - U32 const lowestValid = ms->window.lowLimit; - U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; - U32 const isDictionary = (ms->loadedDictEnd != 0); - U32 const windowLow = isDictionary ? lowestValid : withinWindow; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog); U32 const matchLow = windowLow ? windowLow : 1; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1;