From: inikep Date: Tue, 1 Mar 2016 13:14:35 +0000 (+0100) Subject: fixed repcode before lowLimit X-Git-Tag: v0.6.0^2~17^2~92^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F136%2Fhead;p=thirdparty%2Fzstd.git fixed repcode before lowLimit --- diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 04f5be097..c9847dafa 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -2040,7 +2040,8 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong); - zc->params.searchLength = MINMATCH; + zc->params.searchLength = MINMATCH; /* force ZSTD_btopt to MINMATCH in block mode */ + ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.searchLength); return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0); } @@ -2174,11 +2175,13 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) { + ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, MAX(128 KB, dictSize))); } size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) { + ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin compressionLevel=%d\n", zc->base, compressionLevel); return ZSTD_compressBegin_advanced(zc, NULL, 0, ZSTD_getParams(compressionLevel, 0)); } @@ -2258,11 +2261,13 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) { + ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize)); } size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) { + ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel); return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize)); } diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index a57469866..e9d3bdb69 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -159,7 +159,8 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->dictEnd = NULL; dctx->hufTableX4[0] = HufLog; dctx->flagStaticTables = 0; - dctx->params.searchLength = MINMATCH; + dctx->params.searchLength = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ + ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength); return 0; } @@ -749,7 +750,7 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, } op += 8; match += 8; - if (oMatchEnd > oend-12) { + if (oMatchEnd > oend-(16-3)) { // 3 = MINMATCH if (op < oend_8) { ZSTD_wildcopy(op, match, oend_8 - op); match += oend_8 - op; @@ -857,6 +858,8 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, if (srcSize >= BLOCKSIZE) return ERROR(srcSize_wrong); + ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->params.searchLength); + /* Decode literals sub-block */ litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); if (ZSTD_isError(litCSize)) return litCSize; @@ -966,6 +969,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); + ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength); ZSTD_checkContinuity(dctx, dst); return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); } diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 54f9aaf6b..68b66621f 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -51,8 +51,17 @@ * Common constants ***************************************/ #define ZSTD_OPT_DEBUG 0 // 1 = tableID=0; 3 = price func tests; 5 = check encoded sequences; 9 = full logs -#if ZSTD_OPT_DEBUG > 0 - #include /* for debug */ +#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>0 + #include +#endif +#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 + #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) + #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__) + #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__) +#else + #define ZSTD_LOG_PARSER(...) + #define ZSTD_LOG_ENCODE(...) + #define ZSTD_LOG_BLOCK(...) #endif #define ZSTD_DICT_MAGIC 0xEC30A435 diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 739ef7a61..a95f97b89 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -47,8 +47,7 @@ FORCE_INLINE U32 ZSTD_GETPRICE(seqStore_t* seqStorePtr, U32 litLength, const BYT price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit(seqStorePtr->matchLengthSum+1) - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1); #if ZSTD_OPT_DEBUG >= 3 - switch (seqStorePtr->priceFunc) - { + switch (seqStorePtr->priceFunc) { default: case 0: return 1 + price + seqStorePtr->factor + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)); @@ -557,8 +556,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ml2 = (U32)ZSTD_count(ip, ip-offset, iend); else ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend); - if (offset >= 8) - if (ml2 < mlen || ml2 < MINMATCHOPT) { + if ((offset >= 8) && (ml2 < mlen || ml2 < MINMATCHOPT)) { printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } if (ip < anchor) { printf("%d: ERROR_NoExt ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } @@ -572,7 +570,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ } /* for (cur=0; cur < last_pos; ) */ /* check immediate repcode */ - while ( (anchor <= ilimit) + while ((anchor >= prefixStart + rep_2) && (anchor <= ilimit) && (MEM_readMINMATCH(anchor) == MEM_readMINMATCH(anchor - rep_2)) ) { /* store sequence */ best_mlen = (U32)ZSTD_count(anchor+MINMATCHOPT, anchor+MINMATCHOPT-rep_2, iend); @@ -614,7 +612,8 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, const BYTE* const prefixStart = base + dictLimit; const BYTE* const dictBase = ctx->dictBase; const BYTE* const dictEnd = dictBase + dictLimit; - + const U32 lowLimit = ctx->lowLimit; + U32 rep_2=REPCODE_STARTVALUE, rep_1=REPCODE_STARTVALUE; const U32 maxSearches = 1U << ctx->params.searchLog; const U32 mls = ctx->params.searchLength; @@ -892,21 +891,16 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set #if ZSTD_OPT_DEBUG >= 5 U32 ml2; - if (offset) - { - if (offset > (size_t)(ip - prefixStart)) - { + if (offset) { + if (offset > (size_t)(ip - prefixStart)) { const BYTE* match = dictEnd - (offset - (ip - prefixStart)); ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)offset, dictBase, dictEnd, prefixStart, ip, match); } - else - ml2 = (U32)ZSTD_count(ip, ip-offset, iend); + else ml2 = (U32)ZSTD_count(ip, ip-offset, iend); } - else - ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend); - if (offset >= 8) - if (ml2 < mlen || ml2 < MINMATCHOPT) { + else ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend); + if ((offset >= 8) && (ml2 < mlen || ml2 < MINMATCHOPT)) { printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } if (ip < anchor) { printf("%d: ERROR_Ext ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } @@ -920,7 +914,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set } /* check immediate repcode */ - while (anchor <= ilimit) { + while ((anchor >= base + lowLimit + rep_2) && (anchor <= ilimit)) { const U32 repIndex = (U32)((anchor-base) - rep_2); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; diff --git a/lib/zstd_opt_internal.h b/lib/zstd_opt_internal.h index d643c6926..576f4115c 100644 --- a/lib/zstd_opt_internal.h +++ b/lib/zstd_opt_internal.h @@ -45,17 +45,6 @@ #define ZSTD_FREQ_STEP 1 #define ZSTD_FREQ_DIV 5 -/*- Debug -*/ -#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 - #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) - #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__) - #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__) -#else - #define ZSTD_LOG_PARSER(...) - #define ZSTD_LOG_ENCODE(...) - #define ZSTD_LOG_BLOCK(...) -#endif - typedef struct { U32 off;