From: Yann Collet Date: Wed, 4 Nov 2015 02:52:54 +0000 (+0100) Subject: btlazy2 X-Git-Tag: zstd-0.3.3^2~8 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=96b9f0ba4d22734c4e5b5ee40e807905ab993792;p=thirdparty%2Fzstd.git btlazy2 --- diff --git a/lib/zstdhc.c b/lib/zstdhc.c index 693c713b2..d053b2138 100644 --- a/lib/zstdhc.c +++ b/lib/zstdhc.c @@ -173,14 +173,14 @@ static const U64 prime6bytes = 227718039650203ULL; static size_t ZSTD_HC_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; } static size_t ZSTD_HC_hash6Ptr(const void* p, U32 h) { return ZSTD_HC_hash6(MEM_read64(p), h); } -static size_t ZSTD_HC_hashPtr(const void* p, U32 h, U32 mls) +static size_t ZSTD_HC_hashPtr(const void* p, U32 hBits, U32 mls) { switch(mls) { default: - case 4: return ZSTD_HC_hash4Ptr(p,h); - case 5: return ZSTD_HC_hash5Ptr(p,h); - case 6: return ZSTD_HC_hash6Ptr(p,h); + case 4: return ZSTD_HC_hash4Ptr(p, hBits); + case 5: return ZSTD_HC_hash5Ptr(p, hBits); + case 6: return ZSTD_HC_hash6Ptr(p, hBits); } } @@ -188,8 +188,308 @@ static size_t ZSTD_HC_hashPtr(const void* p, U32 h, U32 mls) /* ************************************* -* HC Compression +* Binary Tree search ***************************************/ +#define BT_SHORTCUT 256 + +/** ZSTD_HC_insertBt1 : add one ptr to tree + @ip : assumed <= iend-8 */ +static void ZSTD_HC_insertBt1(ZSTD_HC_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares) +{ + U32* const hashTable = zc->hashTable; + const U32 hashLog = zc->params.hashLog; + const size_t h = ZSTD_HC_hashPtr(ip, hashLog, mls); + U32* const bt = zc->chainTable; + const U32 btLog = zc->params.chainLog - 1; + const U32 btMask= (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = zc->base; + const U32 current = (U32)(ip-base); + const U32 btLow = btMask >= current ? 0 : current - btMask; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = bt + 2*(current&btMask) + 1; + U32 dummy32; /* to be nullified at the end */ + const U32 windowSize = 1 << zc->params.windowLog; + const U32 windowLow = windowSize >= current ? 0 : current - windowSize; + + hashTable[h] = (U32)(ip-base); /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) + { + U32* nextPtr = bt + 2*(matchIndex & btMask); + const BYTE* match = base + matchIndex; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ + break; /* just drop , to guarantee consistency (miss a bit of compression; if someone knows better, please tell) */ + + if (match[matchLength] < ip[matchLength]) + { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } + else + { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } + } + + *smallerPtr = *largerPtr = 0; +} + + +FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ +size_t ZSTD_HC_insertBtAndFindBestMatch ( + ZSTD_HC_CCtx* zc, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + U32 nbCompares, const U32 mls) +{ + U32* const hashTable = zc->hashTable; + const U32 hashLog = zc->params.hashLog; + const size_t h = ZSTD_HC_hashPtr(ip, hashLog, mls); + U32* const bt = zc->chainTable; + const U32 btLog = zc->params.chainLog - 1; + const U32 btMask= (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = zc->base; + const U32 current = (U32)(ip-base); + const U32 btLow = btMask >= current ? 0 : current - btMask; + const U32 windowSize = 1 << zc->params.windowLog; + const U32 windowLow = windowSize >= current ? 0 : current - windowSize; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = bt + 2*(current&btMask) + 1; + U32 bestLength = 0; + U32 dummy32; /* to be nullified at the end */ + + hashTable[h] = (U32)(ip-base); /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) + { + U32* nextPtr = bt + 2*(matchIndex & btMask); + const BYTE* match = base + matchIndex; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + + if (matchLength > bestLength) + { + bestLength = (U32)matchLength; + *offsetPtr = current - matchIndex; + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ + break; /* drop, next to null, to guarantee consistency (is there a way to do better ?) */ + } + + if (match[matchLength] < ip[matchLength]) + { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } + else + { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } + } + + *smallerPtr = *largerPtr = 0; + + zc->nextToUpdate = current+1; /* current has been inserted */ + if (bestLength < MINMATCH) return 0; + return bestLength; +} + + +static void ZSTD_HC_updateTree(ZSTD_HC_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +{ + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + //size_t dummy; + + for( ; idx < target ; idx++) + ZSTD_HC_insertBt1(zc, base+idx, mls, iend, nbCompares); + //ZSTD_HC_insertBtAndFindBestMatch(zc, base+idx, iend, &dummy, nbCompares, mls); + + zc->nextToUpdate = target; +} + + +/** Tree updater, providing best match */ +FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ +size_t ZSTD_HC_BtFindBestMatch ( + ZSTD_HC_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls) +{ + ZSTD_HC_updateTree(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_HC_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls); +} + + +FORCE_INLINE size_t ZSTD_HC_BtFindBestMatch_selectMLS ( + ZSTD_HC_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_HC_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); + case 5 : return ZSTD_HC_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 6 : return ZSTD_HC_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + } +} + + +size_t ZSTD_HC_compressBlock_btLazy2(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + + size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE; + const U32 maxSearches = 1 << ctx->params.searchLog; + const U32 mls = ctx->params.searchLength; + + /* init */ + ZSTD_resetSeqStore(seqStorePtr); + if (((ip-ctx->base) - ctx->dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; + + /* Match Loop */ + while (ip <= ilimit) + { + size_t matchLength; + size_t offset=999999; + const BYTE* start; + + /* try to find a first match */ + if (MEM_read32(ip) == MEM_read32(ip - offset_2)) + { + /* repcode : we take it*/ + size_t offtmp = offset_2; + size_t litLength = ip - anchor; + matchLength = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_2, iend); + offset_2 = offset_1; + offset_1 = offtmp; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, 0, matchLength); + ip += matchLength+MINMATCH; + anchor = ip; + continue; + } + + offset_2 = offset_1; + matchLength = ZSTD_HC_BtFindBestMatch_selectMLS(ctx, ip, iend, &offset, maxSearches, mls); + if (!matchLength) { ip++; continue; } + + /* let's try to find a better solution */ + start = ip; + + while (ip gain1) + matchLength = ml2, offset = 0, start = ip; + } + { + size_t offset2=999999; + size_t ml2 = ZSTD_HC_BtFindBestMatch_selectMLS(ctx, ip, iend, &offset2, maxSearches, mls); + int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ + int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4); + if (gain2 > gain1) + { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } + } + + /* let's find an even better one */ + if (ip gain1) + matchLength = ml2, offset = 0, start = ip; + } + { + size_t offset2=999999; + size_t ml2 = ZSTD_HC_BtFindBestMatch_selectMLS(ctx, ip, iend, &offset2, maxSearches, mls); + int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ + int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7); + if (gain2 > gain1) + { + matchLength = ml2, offset = offset2, start = ip; + continue; + } + } + } + break; /* nothing found : store previous solution */ + } + + /* store sequence */ + { + size_t litLength = start - anchor; + if (offset) offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH); + ip = start + matchLength; + anchor = ip; + } + + } + + /* Last Literals */ + { + size_t lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } + + /* Final compression stage */ + return ZSTD_compressSequences((BYTE*)dst, maxDstSize, + seqStorePtr, srcSize); +} + + + +/* *********************** +* Hash Chain +*************************/ + /* Update chains up to ip (excluded) */ static U32 ZSTD_HC_insertAndFindFirstIndex (ZSTD_HC_CCtx* zc, const BYTE* ip, U32 mls) { @@ -601,6 +901,8 @@ size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, co return ZSTD_HC_compressBlock_lazy(ctx, dst, maxDstSize, src, srcSize); case ZSTD_HC_lazydeep: return ZSTD_HC_compressBlock_lazydeep(ctx, dst, maxDstSize, src, srcSize); + case ZSTD_HC_btlazy2: + return ZSTD_HC_compressBlock_btLazy2(ctx, dst, maxDstSize, src, srcSize); default : return ERROR(GENERIC); /* unknown block compressor */ } @@ -630,6 +932,9 @@ static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr, case ZSTD_HC_lazydeep: blockCompressor = ZSTD_HC_compressBlock_lazydeep; break; + case ZSTD_HC_btlazy2: + blockCompressor = ZSTD_HC_compressBlock_btLazy2; + break; default : return ERROR(GENERIC); /* unknown block compressor */ } diff --git a/lib/zstdhc_static.h b/lib/zstdhc_static.h index db185b400..36e0781f5 100644 --- a/lib/zstdhc_static.h +++ b/lib/zstdhc_static.h @@ -45,7 +45,7 @@ extern "C" { /* ************************************* * Types ***************************************/ -typedef enum { ZSTD_HC_greedy, ZSTD_HC_lazy, ZSTD_HC_lazydeep } ZSTD_HC_strategy; +typedef enum { ZSTD_HC_greedy, ZSTD_HC_lazy, ZSTD_HC_lazydeep, ZSTD_HC_btlazy2 } ZSTD_HC_strategy; typedef struct { U32 windowLog; /* largest match distance : impact decompression buffer size */ @@ -100,13 +100,13 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] { 19, 13, 17, 3, 5, ZSTD_HC_greedy }, /* level 3 */ { 20, 18, 19, 2, 5, ZSTD_HC_greedy }, /* level 4 */ { 20, 18, 19, 2, 5, ZSTD_HC_lazy }, /* level 5 */ - { 20, 18, 20, 3, 5, ZSTD_HC_lazy }, /* level 6 */ - { 20, 18, 20, 4, 5, ZSTD_HC_lazy }, /* level 7 */ + { 20, 18, 19, 2, 5, ZSTD_HC_lazydeep }, //{ 20, 18, 20, 3, 5, ZSTD_HC_lazy }, /* level 6 */ + { 20, 18, 19, 2, 5, ZSTD_HC_btlazy2 }, //{ 20, 18, 20, 4, 5, ZSTD_HC_lazy }, /* level 7 */ { 21, 19, 20, 4, 5, ZSTD_HC_lazy }, /* level 8 */ { 21, 19, 20, 5, 5, ZSTD_HC_lazy }, /* level 9 */ { 21, 20, 20, 5, 5, ZSTD_HC_lazy }, /* level 10 */ { 21, 20, 20, 5, 5, ZSTD_HC_lazydeep }, /* level 11 */ - { 22, 20, 22, 5, 5, ZSTD_HC_lazydeep }, /* level 12 */ + { 21, 20, 20, 5, 5, ZSTD_HC_btlazy2 }, //{ 22, 20, 22, 5, 5, ZSTD_HC_lazydeep }, /* level 12 */ { 22, 20, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 13 */ { 21, 21, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 14 */ { 22, 21, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 15 */ diff --git a/programs/bench.c b/programs/bench.c index 1892ddd60..c19d0fd97 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -83,7 +83,7 @@ #define MB *(1 <<20) #define GB *(1U<<30) -static const size_t maxMemory = sizeof(size_t)==4 ? (2 GB - 64 MB) : (size_t)(1ULL << (sizeof(size_t)*8-31)); +static const size_t maxMemory = sizeof(size_t)==4 ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); #define DEFAULT_CHUNKSIZE (4 MB) static U32 g_compressibilityDefault = 50; diff --git a/programs/paramgrill.c b/programs/paramgrill.c index d27a22b5f..78f2b1959 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -425,7 +425,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, return 0; } -const char* g_stratName[] = { "ZSTD_HC_greedy ", "ZSTD_HC_lazy ", "ZSTD_HC_lazydeep" }; +const char* g_stratName[] = { "ZSTD_HC_greedy ", "ZSTD_HC_lazy ", "ZSTD_HC_lazydeep", "ZSTD_HC_btlazy2 " }; static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_parameters params, size_t srcSize) { @@ -576,7 +576,7 @@ static BYTE g_alreadyTested[ZSTD_HC_WINDOWLOG_MAX+1-ZSTD_HC_WINDOWLOG_MIN] [ZSTD_HC_HASHLOG_MAX+1-ZSTD_HC_HASHLOG_MIN] [ZSTD_HC_SEARCHLOG_MAX+1-ZSTD_HC_SEARCHLOG_MIN] [ZSTD_HC_SEARCHLENGTH_MAX+1-ZSTD_HC_SEARCHLENGTH_MIN] - [3 /* strategy */ ] = {}; /* init to zero */ + [4 /* strategy */ ] = {}; /* init to zero */ #define NB_TESTS_PLAYED(p) \ g_alreadyTested[p.windowLog-ZSTD_HC_WINDOWLOG_MIN] \ @@ -647,7 +647,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, if (p.searchLength > ZSTD_HC_SEARCHLENGTH_MAX) continue; if (p.searchLength < ZSTD_HC_SEARCHLENGTH_MIN) continue; if (p.strategy < ZSTD_HC_greedy) continue; - if (p.strategy > ZSTD_HC_lazydeep) continue; + if (p.strategy > ZSTD_HC_btlazy2) continue; /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) @@ -680,7 +680,7 @@ static void BMK_selectRandomStart( p.searchLog = FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLOG_MAX+1 - ZSTD_HC_SEARCHLOG_MIN) + ZSTD_HC_SEARCHLOG_MIN; p.windowLog = FUZ_rand(&g_rand) % (ZSTD_HC_WINDOWLOG_MAX+1 - ZSTD_HC_WINDOWLOG_MIN) + ZSTD_HC_WINDOWLOG_MIN; p.searchLength=FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLENGTH_MAX+1 - ZSTD_HC_SEARCHLENGTH_MIN) + ZSTD_HC_SEARCHLENGTH_MIN; - p.strategy = (ZSTD_HC_strategy) (FUZ_rand(&g_rand) % 3); + p.strategy = (ZSTD_HC_strategy) (FUZ_rand(&g_rand) % 4); playAround(f, winners, p, srcBuffer, srcSize, ctx); } else