From: Yann Collet Date: Wed, 9 May 2018 22:46:11 +0000 (-0700) Subject: opt: minor improvements to log traces X-Git-Tag: v1.3.5~3^2~55^2~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ac6105463a69d48652e28e1919ea326d4151dd53;p=thirdparty%2Fzstd.git opt: minor improvements to log traces slight improvement when using fractional-bit evaluation (opt:dictionay) --- diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 80044b9c2..8e170150f 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -161,10 +161,10 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi } } } } #if 0 /* debug : symbol costs */ - DEBUGLOG(2, "\n --- table statistics : "); + DEBUGLOG(5, "\n --- table statistics : "); { U32 symbol; for (symbol=0; symbol<=maxSymbolValue; symbol++) { - DEBUGLOG(2, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", + DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", symbol, normalizedCounter[symbol], FSE_getMaxNbBits(symbolTT, symbol), (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d69b01773..d6e3e6b07 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3450,7 +3450,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 22, 21, 22, 4, 5, 48, ZSTD_btopt }, /* level 16 */ { 23, 22, 22, 4, 4, 48, ZSTD_btopt }, /* level 17 */ { 23, 22, 22, 5, 3, 64, ZSTD_btopt }, /* level 18 */ - { 23, 23, 22, 7, 3,128, ZSTD_btultra }, /* level 19 */ + { 23, 23, 22, 7, 3,128, ZSTD_btopt }, /* level 19 */ { 25, 25, 23, 7, 3,128, ZSTD_btultra }, /* level 20 */ { 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */ { 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */ @@ -3476,7 +3476,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/ { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ - { 18, 19, 18, 9, 3,256, ZSTD_btultra }, /* level 19.*/ + { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ { 18, 19, 18, 11, 3,512, ZSTD_btultra }, /* level 20.*/ { 18, 19, 18, 12, 3,512, ZSTD_btultra }, /* level 21.*/ { 18, 19, 18, 13, 3,512, ZSTD_btultra }, /* level 22.*/ @@ -3502,7 +3502,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 17, 18, 17, 7, 3, 32, ZSTD_btopt }, /* level 16.*/ { 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/ - { 17, 18, 17, 7, 3,256, ZSTD_btultra }, /* level 19.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/ { 17, 18, 17, 9, 3,256, ZSTD_btultra }, /* level 20.*/ { 17, 18, 17, 10, 3,256, ZSTD_btultra }, /* level 21.*/ { 17, 18, 17, 11, 3,512, ZSTD_btultra }, /* level 22.*/ @@ -3528,10 +3528,10 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/ { 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ { 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/ - { 14, 15, 15, 6, 3,256, ZSTD_btultra }, /* level 19.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/ { 14, 15, 15, 8, 3,256, ZSTD_btultra }, /* level 20.*/ { 14, 15, 15, 9, 3,256, ZSTD_btultra }, /* level 21.*/ - { 14, 15, 15, 10, 3,256, ZSTD_btultra }, /* level 22.*/ + { 14, 15, 15, 10, 3,512, ZSTD_btultra }, /* level 22.*/ }, }; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 73ecda721..486c86931 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -101,6 +101,12 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr, # define BITCOST_SYMBOL(t,l,s) FSE_bitCost(t,l,s,BITCOST_ACCURACY) #endif +MEM_STATIC double +ZSTD_fCost(U32 price) +{ + return (double)price / (BITCOST_MULTIPLIER*8); +} + /* ZSTD_rawLiteralsCost() : * cost of literals (only) in specified segment (which length can be 0). * does not include cost of literalLength symbol */ @@ -433,7 +439,7 @@ void ZSTD_updateTree_internal( const BYTE* const base = ms->window.base; U32 const target = (U32)(ip - base); U32 idx = ms->nextToUpdate; - DEBUGLOG(7, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)", + DEBUGLOG(8, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)", idx, target, extDict); while(idx < target) @@ -481,7 +487,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 nbCompares = 1U << cParams->searchLog; size_t bestLength = lengthToBeat-1; - DEBUGLOG(7, "ZSTD_insertBtAndGetAllMatches"); + DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches"); /* check repCode */ { U32 const lastR = ZSTD_REP_NUM + ll0; @@ -612,7 +618,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( ZSTD_match_t* matches, U32 const lengthToBeat) { U32 const matchLengthSearch = cParams->searchLength; - DEBUGLOG(7, "ZSTD_BtGetAllMatches"); + DEBUGLOG(8, "ZSTD_BtGetAllMatches"); if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ ZSTD_updateTree_internal(ms, cParams, ip, iHighLimit, matchLengthSearch, extDict); switch(matchLengthSearch) @@ -786,8 +792,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, for ( ; pos <= end ; pos++ ) { U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel); U32 const sequencePrice = literalsPrice + matchPrice; - DEBUGLOG(7, "rPos:%u => set initial price : %u", - pos, sequencePrice); + DEBUGLOG(7, "rPos:%u => set initial price : %.2f", + pos, ZSTD_fCost(sequencePrice)); opt[pos].mlen = pos; opt[pos].off = offset; opt[pos].litlen = litlen; @@ -814,14 +820,18 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, } assert(price < 1000000000); /* overflow check */ if (price <= opt[cur].price) { - DEBUGLOG(7, "rPos:%u : better price (%u<%u) using literal", - cur, price, opt[cur].price); + DEBUGLOG(7, "rPos:%u : better price (%.2f<=%.2f) using literal", + cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price)); opt[cur].mlen = 1; opt[cur].off = 0; opt[cur].litlen = litlen; opt[cur].price = price; memcpy(opt[cur].rep, opt[cur-1].rep, sizeof(opt[cur].rep)); - } } + } else { + DEBUGLOG(7, "rPos:%u : literal would cost more (%.2f>%.2f)", + cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price)); + } + } /* last match must start at a minimum distance of 8 from oend */ if (inr > ilimit) continue; @@ -829,8 +839,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, if (cur == last_pos) break; if ( (optLevel==0) /*static_test*/ - && (opt[cur+1].price <= opt[cur].price) ) + && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { + DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1); continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ + } { U32 const ll0 = (opt[cur].mlen != 1); U32 const litlen = (opt[cur].mlen == 1) ? opt[cur].litlen : 0; @@ -838,7 +850,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const basePrice = previousPrice + ZSTD_fullLiteralsCost(inr-litlen, litlen, optStatePtr); U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, inr, iend, extDict, opt[cur].rep, ll0, matches, minMatch); U32 matchNb; - if (!nbMatches) continue; + if (!nbMatches) { + DEBUGLOG(7, "rPos:%u : no match found", cur); + continue; + } { U32 const maxML = matches[nbMatches-1].len; DEBUGLOG(7, "rPos:%u, found %u matches, of maxLength=%u", @@ -868,8 +883,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); if ((pos > last_pos) || (price < opt[pos].price)) { - DEBUGLOG(7, "rPos:%u => new better price (%u<%u)", - pos, price, opt[pos].price); + DEBUGLOG(7, "rPos:%u => new better price (%.2f<%.2f)", + pos, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */ opt[pos].mlen = mlen; opt[pos].off = offset;