From: inikep Date: Wed, 24 Aug 2016 15:19:12 +0000 (+0200) Subject: removed stats in debug mode X-Git-Tag: v1.0.0^2~35^2~4 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=8a36f8527cec8c9470640afef07e4c3222c63c85;p=thirdparty%2Fzstd.git removed stats in debug mode --- diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 7dc843e35..fc07a79da 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -51,18 +51,9 @@ /*-************************************* * Common constants ***************************************/ -#define ZSTD_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ -#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 - #include - #include - #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) - #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__) - #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__) -#else - #define ZSTD_LOG_PARSER(...) - #define ZSTD_LOG_ENCODE(...) - #define ZSTD_LOG_BLOCK(...) -#endif +#define ZSTD_LOG_PARSER(...) +#define ZSTD_LOG_ENCODE(...) +#define ZSTD_LOG_BLOCK(...) #define ZSTD_OPT_NUM (1<<12) #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ @@ -184,16 +175,6 @@ typedef struct { U32 rep[ZSTD_REP_NUM]; } ZSTD_optimal_t; -#if ZSTD_OPT_DEBUG == 3 - #include ".debug/zstd_stats.h" -#else - struct ZSTD_stats_s { U32 unused; }; - MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } - MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -#endif /* #if ZSTD_OPT_DEBUG == 3 */ - typedef struct seqDef_s { U32 offset; diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h deleted file mode 100644 index 5cefa3115..000000000 --- a/lib/compress/.debug/zstd_stats.h +++ /dev/null @@ -1,162 +0,0 @@ -/* - zstd - standard compression library - Header File for static linking only - Copyright (C) 2014-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ -#ifndef ZSTD_STATS_H -#define ZSTD_STATS_H - - -#if defined (__cplusplus) -extern "C" { -#endif - - -/*-************************************* -* Types -***************************************/ -struct ZSTD_stats_s { - U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength; - U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum; - U32 litSum, matchLengthSum, litLengthSum, offCodeSum; - U32 matchLengthFreq[MaxML+1]; - U32 litLengthFreq[MaxLL+1]; - U32 litFreq[1<totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); - printf("\navgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); - printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8); -} - - -MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) -{ - stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1; - stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = 0; -} - - -MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) -{ - unsigned u; - - stats->litSum = (2<litLengthSum = MaxLL+1; - stats->matchLengthSum = MaxML+1; - stats->offCodeSum = (MaxOff+1); - - for (u=0; u<=MaxLit; u++) - stats->litFreq[u] = 1; - for (u=0; u<=MaxLL; u++) - stats->litLengthFreq[u] = 1; - for (u=0; u<=MaxML; u++) - stats->matchLengthFreq[u] = 1; - for (u=0; u<=MaxOff; u++) - stats->offCodeFreq[u] = 1; -} - - -MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) -{ - U32 u; - /* literals */ - stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); - for (u=0; u < litLength; u++) - stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); - stats->litSum += litLength; - for (u=0; u < litLength; u++) - stats->litFreq[literals[u]]++; - - /* literal Length */ - { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24 }; - const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; - if (litLength) { - stats->priceLitLength += LL_bits[llCode] + ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[llCode]+1); - } else { - stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); - } - stats->litLengthFreq[llCode]++; - stats->litLengthSum++; - } - - /* match offset */ - { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); - stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); - stats->priceOffset += offCode; - stats->offCodeSum++; - stats->offCodeFreq[offCode]++; - } - - /* match Length */ - { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; - const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; - stats->priceMatchLength += ML_bits[mlCode] + ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[mlCode]+1); - stats->matchLengthFreq[mlCode]++; - stats->matchLengthSum++; - } - - if (offset == 0) stats->totalRepSum++; - stats->totalSeqSum++; - stats->totalMatchSum += matchLength; - stats->totalLitSum += litLength; -} - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_STATIC_H */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a696b2bc6..826c5f97a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -765,8 +765,6 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif - ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, (const BYTE*)literals, offsetCode, matchCode); /* debug only */ - /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); seqStorePtr->lit += litLength; @@ -1945,7 +1943,6 @@ _storeSequence: { size_t const lastLLSize = iend - anchor; memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; - ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); } } @@ -2236,7 +2233,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, BYTE* op = ostart; U32 const maxDist = 1 << cctx->params.cParams.windowLog; ZSTD_stats_t* stats = &cctx->seqStore.stats; - ZSTD_statsInit(stats); /* debug only */ if (cctx->params.fParams.checksumFlag) XXH64_update(&cctx->xxhState, src, srcSize); @@ -2244,7 +2240,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, while (remaining) { U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); size_t cSize; - ZSTD_statsResetFreqs(stats); /* debug only */ if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; @@ -2293,7 +2288,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, } if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; - ZSTD_statsPrint(stats, cctx->params.cParams.searchLength); /* debug only */ return op-ostart; } diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 612ac3b4d..1a37101a7 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -295,23 +295,10 @@ static U32 ZSTD_insertBtAndGetAllMatches ( if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { match = base + matchIndex; if (match[matchLength] == ip[matchLength]) { -#if ZSTD_OPT_DEBUG >= 5 - size_t ml; - if (matchIndex < dictLimit) - ml = ZSTD_count_2segments(ip, dictBase + matchIndex, iLimit, dictEnd, prefixStart); - else - ml = ZSTD_count(ip, match, ip+matchLength); - if (ml < matchLength) - printf("%d: ERROR_NOEXT: offset=%d matchLength=%d matchIndex=%d dictLimit=%d ml=%d\n", current, (int)(current - matchIndex), (int)matchLength, (int)matchIndex, (int)dictLimit, (int)ml), exit(0); -#endif matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1; } } else { match = dictBase + matchIndex; -#if ZSTD_OPT_DEBUG >= 5 - if (memcmp(match, ip, matchLength) != 0) - printf("%d: ERROR_EXT: matchLength=%d ZSTD_count=%d\n", current, (int)matchLength, (int)ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart)), exit(0); -#endif matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); ZSTD_LOG_PARSER("%d: ZSTD_INSERTBTANDGETALLMATCHES=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)matchLength, (int)(current - matchIndex), dictBase, dictEnd, prefixStart, ip, match); if (matchIndex+matchLength >= dictLimit) @@ -672,20 +659,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); -#if ZSTD_OPT_DEBUG >= 5 - U32 ml2; - if (offset+1 > ZSTD_REP_MOVE_OPT) - ml2 = (U32)ZSTD_count(ip, ip-(offset+1-ZSTD_REP_MOVE_OPT), iend); - else - ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); - if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { - printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } - if (ip < anchor) { - printf("%d: ERROR_NoExt ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } - if (ip + mlen > iend) { - printf("%d: ERROR_NoExt ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } -#endif - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); anchor = ip = ip + mlen; @@ -990,26 +963,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); -#if ZSTD_OPT_DEBUG >= 5 - U32 ml2; - if (offset+1 > ZSTD_REP_MOVE_OPT) { - best_off = offset+1 - ZSTD_REP_MOVE_OPT; - if (best_off > (size_t)(ip - prefixStart)) { - const BYTE* match = dictEnd - (best_off - (ip - prefixStart)); - ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); - ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)best_off, dictBase, dictEnd, prefixStart, ip, match); - } - else ml2 = (U32)ZSTD_count(ip, ip-best_off, iend); - } - else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); - if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { - printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } - if (ip < anchor) { - printf("%d: ERROR_Ext ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } - if (ip + mlen > iend) { - printf("%d: ERROR_Ext ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } -#endif - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); anchor = ip = ip + mlen;