From: inikep Date: Tue, 5 Apr 2016 19:03:43 +0000 (+0200) Subject: ZSTD_compressBlock_opt_extDict_generic in progress X-Git-Tag: v0.6.0^2~17^2~16^2~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5ce00aee37fda7eec60b97dd5b110fcb0f6ddb93;p=thirdparty%2Fzstd.git ZSTD_compressBlock_opt_extDict_generic in progress --- diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index e4009a3a7..544323ce3 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1671,7 +1671,7 @@ void ZSTD_compressBlock_greedy_generic(ZSTD_CCtx* ctx, /* catch up */ while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE])) /* only search for offset within prefix */ { start--; matchLength++; } - rep[1] = rep[0]; rep[0] = offset - ZSTD_REP_MOVE; + rep[1] = rep[0]; rep[0] = (U32)(offset - ZSTD_REP_MOVE); _storeSequence: /* store sequence */ @@ -1685,7 +1685,7 @@ _storeSequence: && (MEM_read32(ip) == MEM_read32(ip - rep[1])) ) { /* store sequence */ matchLength = ZSTD_count(ip+MINMATCH, ip+MINMATCH-rep[1], iend); - offset = rep[1]; rep[1] = rep[0]; rep[0] = offset; /* swap offset history */ + offset = rep[1]; rep[1] = rep[0]; rep[0] = (U32)offset; /* swap offset history */ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength); ip += matchLength+MINMATCH; anchor = ip; @@ -1767,7 +1767,7 @@ void ZSTD_compressBlock_greedy_extDict_generic(ZSTD_CCtx* ctx, const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } - rep[1] = rep[0]; rep[0] = offset - ZSTD_REP_MOVE; + rep[1] = rep[0]; rep[0] = (U32)(offset - ZSTD_REP_MOVE); } _storeSequence: @@ -1787,7 +1787,7 @@ _storeSequence: /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH; - offset = rep[1]; rep[1] = rep[0]; rep[0] = offset; /* swap offset history */ + offset = rep[1]; rep[1] = rep[0]; rep[0] = (U32)offset; /* swap offset history */ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; @@ -1925,7 +1925,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, if (offset >= ZSTD_REP_NUM) { rep[2] = rep[1]; rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE; + rep[0] = (U32)(offset - ZSTD_REP_MOVE); } else { if (offset != 0) { U32 temp = rep[offset]; @@ -1955,7 +1955,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 2); + ZSTD_compressBlock_opt_generic(ctx, src, srcSize); } static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) @@ -2129,7 +2129,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, if (offset >= ZSTD_REP_NUM) { rep[2] = rep[1]; rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE; + rep[0] = (U32)(offset - ZSTD_REP_MOVE); } else { if (offset != 0) { U32 temp = rep[offset]; @@ -2175,7 +2175,7 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 2); + ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize); } diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 44fc5c3c0..7b59399d2 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -421,8 +421,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( *********************************/ FORCE_INLINE void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize, - const U32 depth) + const void* src, size_t srcSize) { seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const istart = (const BYTE*)src; @@ -473,14 +472,14 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, if (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch)) { /* repcode : we take it */ mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch; - best_off = (i<=1 && ip == anchor) ? 1-i : i; ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep[i], (int)mlen); - if (depth==0 || mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; goto _storeSequence; } + best_off = (i<=1 && ip == anchor) ? 1-i : i; litlen = opt[0].litlen; do { price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - minMatch); @@ -548,7 +547,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - if (opt[cur].off >= ZSTD_REP_NUM) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; @@ -562,7 +560,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); - best_mlen = 0; for (U32 i=0; i sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; @@ -580,6 +575,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, goto _storeSequence; } + best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i; if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) { @@ -620,7 +616,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); - while (mlen <= best_mlen) { if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; @@ -634,7 +629,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); - if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); @@ -686,10 +680,10 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ rep[0] = offset - ZSTD_REP_MOVE; } else { if (offset != 0) { - U32 temp = rep[offset]; + best_off = rep[offset]; if (offset != 1) rep[2] = rep[1]; rep[1] = rep[0]; - rep[0] = temp; + rep[0] = best_off; } if (litLength == 0 && offset<=1) offset = 1-offset; @@ -714,8 +708,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-minMatch); anchor = ip = ip + mlen; - } /* for (cur=0; cur < last_pos; ) */ - } + } } /* for (cur=0; cur < last_pos; ) */ { /* Last Literals */ size_t lastLLSize = iend - anchor; @@ -728,12 +721,8 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ FORCE_INLINE void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize, - const U32 depth) + const void* src, size_t srcSize) { - printf("NOT IMPLEMENTED: ZSTD_compressBlock_opt_extDict_generic\n"), exit(0); - (void)ctx; (void)src; (void)srcSize; (void)depth; (void)ZSTD_BtGetAllMatches_selectMLS_extDict; -#if 0 seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; @@ -746,7 +735,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const prefixStart = base + dictLimit; const BYTE* const dictBase = ctx->dictBase; const BYTE* const dictEnd = dictBase + dictLimit; - const U32 lowLimit = ctx->lowLimit; const U32 maxSearches = 1U << ctx->params.cParams.searchLog; const U32 sufficient_len = ctx->params.cParams.targetLength; @@ -782,27 +770,28 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, opt[0].litlen = (U32)(ip - litstart); /* check repCode */ - { - const U32 repIndex = (U32)(current+1 - rep_1); + for (U32 i=0; i= 3) /* intentional overflow */ - && (MEM_readMINMATCH(ip+1, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(ip+1+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep_1, (int)mlen); - if (depth==0 || mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { - ip+=1; best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1; + ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep[0], (int)mlen); + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; goto _storeSequence; } - litlen = opt[0].litlen + 1; + best_off = (i<=1 && ip == anchor) ? 1-i : i; + litlen = opt[0].litlen; do { - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); - if (mlen + 1 > last_pos || price < opt[mlen + 1].price) - SET_PRICE(mlen + 1, mlen, 0, litlen, price); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - minMatch); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); mlen--; } while (mlen >= minMatch); } } @@ -814,8 +803,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } - opt[0].rep = rep_1; - opt[0].rep2 = rep_2; + for (int i=0; i sufficient_len) { @@ -846,96 +835,84 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* check further positions */ for (cur = 1; cur <= last_pos; cur++) { - size_t cur_rep; - inr = ip + cur; + inr = ip + cur; - if (opt[cur-1].mlen == 1) { + if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; if (cur > litlen) { price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); } else price = ZSTD_getLiteralPrice(seqStorePtr, litlen, litstart); - } else { + } else { litlen = 1; price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); - } + } - if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) SET_PRICE(cur, 1, 0, litlen, price); - if (cur == last_pos) break; + if (cur == last_pos) break; - if (inr > ilimit) // last match must start at a minimum distance of 8 from oend + if (inr > ilimit) // last match must start at a minimum distance of 8 from oend continue; - mlen = opt[cur].mlen; - - if (opt[cur].off) { - opt[cur].rep2 = opt[cur-mlen].rep; - opt[cur].rep = opt[cur].off; - ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } else { - if (cur!=mlen && opt[cur].litlen == 0) { - opt[cur].rep2 = opt[cur-mlen].rep; - opt[cur].rep = opt[cur-mlen].rep2; - ZSTD_LOG_ENCODE("%d: COPYREP_SWI cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } else { - opt[cur].rep2 = opt[cur-mlen].rep2; - opt[cur].rep = opt[cur-mlen].rep; - ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } } - - ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); - - best_mlen = 0; + mlen = opt[cur].mlen; + if (opt[cur].off >= ZSTD_REP_NUM) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; + ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); + } - if (opt[cur].mlen != 1) { - cur_rep = opt[cur].rep2; - ZSTD_LOG_PARSER("%d: tryExt REP2 rep2=%u mlen=%u\n", (int)(inr-base), (U32)cur_rep, mlen); - } else { - cur_rep = opt[cur].rep; - ZSTD_LOG_PARSER("%d: tryExt REP1 rep=%u mlen=%u\n", (int)(inr-base), (U32)cur_rep, mlen); - } + ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); + + best_mlen = 0; + + for (U32 i=0; i= 3) /* intentional overflow */ + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), mlen, 0, opt[cur].rep, cur, opt[cur].off); + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; + best_off = i; + ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); + last_pos = cur + 1; + goto _storeSequence; + } - const U32 repIndex = (U32)(current+cur - cur_rep); - const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { - /* repcode detected */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), mlen, 0, opt[cur].rep, cur, opt[cur].off); + best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i; + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - minMatch); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - minMatch); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - minMatch); + } - if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; - best_off = 0; - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); - last_pos = cur + 1; - goto _storeSequence; - } - - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, 0, mlen - minMatch); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, 0, mlen - minMatch); - } + ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); - best_mlen = mlen; - - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, 0, price, litlen); - - do { - if (cur + mlen > last_pos || price <= opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH - SET_PRICE(cur + mlen, mlen, 0, litlen, price); - mlen--; - } while (mlen >= minMatch); - } + do { + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; @@ -1013,19 +990,24 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set cur += mlen; litLength = (U32)(ip - anchor); - ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2); + ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - if (offset) { - rep_2 = rep_1; - rep_1 = offset; + if (offset >= ZSTD_REP_NUM) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE; } else { - if (litLength == 0) { - best_off = rep_2; - rep_2 = rep_1; - rep_1 = best_off; - } } + if (offset != 0) { + best_off = rep[offset]; + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + + if (litLength == 0 && offset<=1) offset = 1-offset; + } - ZSTD_LOG_ENCODE("%d/%d: ENCODE2 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2); + ZSTD_LOG_ENCODE("%d/%d: ENCODE2 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); #if ZSTD_OPT_DEBUG >= 5 U32 ml2; @@ -1037,7 +1019,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set } else ml2 = (U32)ZSTD_count(ip, ip-offset, iend); } - else ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend); + else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } if (ip < anchor) { @@ -1049,54 +1031,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-minMatch); anchor = ip = ip + mlen; - } - -#if 0 - /* check immediate repcode */ - while ((anchor >= base + lowLimit + rep_2) && (anchor <= ilimit)) { - if ((anchor - rep_2) >= prefixStart) { - if (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(anchor - rep_2, minMatch)) - mlen = (U32)ZSTD_count(anchor+minMatch, anchor - rep_2 + minMatch, iend) + minMatch; - else - break; - } else { - const BYTE* repMatch = dictBase + ((anchor-base) - rep_2); - if ((repMatch + minMatch <= dictEnd) && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(repMatch, minMatch))) - mlen = (U32)ZSTD_count_2segments(anchor+minMatch, repMatch+minMatch, iend, dictEnd, prefixStart) + minMatch; - else - break; - } - - offset = rep_2; rep_2 = rep_1; rep_1 = offset; /* swap offset history */ - ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2); - ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-minMatch); - anchor += mlen; - } -#else - /* check immediate repcode */ - /* minimal correctness condition = while ((anchor >= prefixStart + REPCODE_STARTVALUE) && (anchor <= ilimit)) { */ - while ((anchor >= base + lowLimit + rep_2) && (anchor <= ilimit)) { - const U32 repIndex = (U32)((anchor-base) - rep_2); - const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { - /* repcode detected, let's take it */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(anchor+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - offset = rep_2; rep_2 = rep_1; rep_1 = offset; /* swap offset history */ - ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2); - ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-minMatch); - anchor += mlen; - continue; /* faster when present ... (?) */ - } - break; - } -#endif - if (anchor > ip) ip = anchor; - } + } } { /* Last Literals */ size_t lastLLSize = iend - anchor; @@ -1104,5 +1039,4 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; } -#endif } diff --git a/programs/Makefile b/programs/Makefile index 6aca9768e..53077406d 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -84,6 +84,10 @@ default: zstd all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen +zstd_clang: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ + zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c + clang.exe --target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) + zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)