#define ZSTD_LOG_TRY_PRICE(fmt, args...) ;//printf(fmt, ##args)
#define ZSTD_OPT_NUM (1<<12)
-
+#define ZSTD_FREQ_THRESHOLD (256)
const int tab32[32] = {
0, 9, 1, 10, 13, 21, 2, 29,
FORCE_INLINE size_t ZSTD_getLiteralPriceReal(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals)
{
- size_t freq;
size_t price = 0;
- size_t litBits, litLenBits;
- // printf("litSum=%d litLengthSum=%d matchLengthSum=%d offCodeSum=%d\n", seqStorePtr->litSum, seqStorePtr->litLengthSum, seqStorePtr->matchLengthSum, seqStorePtr->offCodeSum);
- /* literals */
- litBits = 0;
if (litLength > 0) {
+ /* literals */
for (int i=litLength-1; i>=0; i--)
-// litBits += -log2((double)seqStorePtr->litFreq[literals[i]]/(double)seqStorePtr->litSum);
- litBits += log2_32(seqStorePtr->litSum) - log2_32(seqStorePtr->litFreq[literals[i]]);
+ price += log2_32(seqStorePtr->litSum) - log2_32(seqStorePtr->litFreq[literals[i]]);
/* literal Length */
+ size_t freq;
if (litLength >= MaxLL) {
freq = seqStorePtr->litLengthFreq[MaxLL];
if (litLength<255 + MaxLL) {
price += 8;
if (litLength < (1<<15)) price += 16; else price += 24;
} }
- else freq = seqStorePtr->litLengthFreq[litLength];
-// litLenBits = -log2((double)freq/(double)seqStorePtr->litLengthSum);
- litLenBits = log2_32(seqStorePtr->litLengthSum) - log2_32(freq);
+ else
+ freq = seqStorePtr->litLengthFreq[litLength];
+ price += log2_32(seqStorePtr->litLengthSum) - log2_32(freq);
}
- else litLenBits = 0;
-// freq = round(1.0f*(litBits + litLenBits + price));
- freq = litBits + litLenBits + price;
-// printf("litLength=%d litBits=%.02f litLenBits=%.02f dumpsPrice=%d sum=%d\n", (int)litLength, litBits, litLenBits, (int)price, (int)freq);
-
- // printf("old=%d new=%d\n", (int)((litLength<<3)+1), (int)freq/8);
- if (freq <= 0) freq = 1;
- return freq;
+ return price + (price <= 0);
}
FORCE_INLINE size_t ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals)
{
-#if 1
- return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals);
-#else
- size_t lit_cost = 1 + (litLength<<3);
- return lit_cost;
-#endif
+ if (seqStorePtr->litSum > ZSTD_FREQ_THRESHOLD)
+ return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals);
+
+ return 1 + (litLength<<3);
}
FORCE_INLINE size_t ZSTD_getMatchPriceReal(seqStore_t* seqStorePtr, size_t offset, size_t matchLength)
{
- size_t freq;
- size_t price = 0;
- size_t offCodeBits, matchBits;
- // printf("litSum=%d litLengthSum=%d matchLengthSum=%d offCodeSum=%d\n", seqStorePtr->litSum, seqStorePtr->litLengthSum, seqStorePtr->matchLengthSum, seqStorePtr->offCodeSum);
-
/* match offset */
BYTE offCode = (BYTE)ZSTD_highbit(offset) + 1;
if (offset==0)
offCode = 0;
- // offCodeBits = -log2((double)seqStorePtr->offCodeFreq[offCode]/(double)seqStorePtr->offCodeSum);
- offCodeBits = log2_32(seqStorePtr->offCodeSum) - log2_32(seqStorePtr->offCodeFreq[offCode]);
- // printf("offCodeBits=%.02f matchBits=%.02f dumpsPrice=%d sum=%d\n", offCodeBits, matchBits, (int)price, (int)freq);
-
- offCodeBits += offCode;
+ size_t price = log2_32(seqStorePtr->offCodeSum) - log2_32(seqStorePtr->offCodeFreq[offCode]);
+ price += offCode;
/* match Length */
+ size_t freq;
if (matchLength >= MaxML) {
freq = seqStorePtr->matchLengthFreq[MaxML];
if (matchLength < 255+MaxML) {
if (matchLength < (1<<15)) price += 16; else price += 24;
} }
else freq = seqStorePtr->matchLengthFreq[matchLength];
-// matchBits = -log2((double)freq/(double)seqStorePtr->matchLengthSum);
- matchBits = log2_32(seqStorePtr->matchLengthSum) - log2_32(freq);
+ price += log2_32(seqStorePtr->matchLengthSum) - log2_32(freq);
-// freq = round(1.0f*(offCodeBits + matchBits + price));
- freq = offCodeBits + matchBits + price;
-// printf("offCodeBits=%.02f matchBits=%.02f dumpsPrice=%d sum=%d\n", offCodeBits, matchBits, (int)price, (int)freq);
- return freq;
+ return price;
}
FORCE_INLINE size_t ZSTD_getPrice(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
{
-#if 1
- size_t lit_cost = ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals);
- // size_t match_cost_old = ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
- size_t match_cost = ZSTD_getMatchPriceReal(seqStorePtr, offset, matchLength);
- return lit_cost + match_cost;
-#else
- size_t lit_cost = (litLength<<3);
- size_t match_cost = ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
- return lit_cost + match_cost;
-#endif
+ if (seqStorePtr->litSum > ZSTD_FREQ_THRESHOLD)
+ return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals) + ZSTD_getMatchPriceReal(seqStorePtr, offset, matchLength);
+
+ return (litLength<<3) + ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
}
U32 dummy32; /* to be nullified at the end */
size_t mnum = 0;
-#if 1
bestLength = 0;
-#else
- bestLength--;
-#endif
hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow)) {
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
}
-#if 1
if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */
}
-#else
- if (matchLength > matchEndIdx - matchIndex)
- matchEndIdx = matchIndex + (U32)matchLength;
-
- if (matchLength > bestLength) {
- bestLength = matchLength;
- matches[mnum].off = current - matchIndex;
- matches[mnum].len = matchLength;
- matches[mnum].back = 0;
- mnum++;
-
- if (matchLength > ZSTD_OPT_NUM) break;
- }
-
- if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
- break; /* drop, to guarantee consistency (miss a little bit of compression) */
-#endif
-
if (match[matchLength] < ip[matchLength]) {
/* match is smaller than current */
/* *******************************
-* Optimal parser OLD
+* Old parser
*********************************/
FORCE_INLINE
void ZSTD_compressBlock_opt2_generic(ZSTD_CCtx* ctx,
last_pos = 0;
inr = ip;
opt[0].litlen = ip - anchor;
- // opt[0].price = ZSTD_getLiteralPrice(seqStorePtr, opt[0].litlen, anchor);
-
+
+
/* check repCode */
if (MEM_read32(ip+1) == MEM_read32(ip+1 - rep_1)) {
/* repcode : we take it */
if (faster_get_matches && last_pos)
match_num = 0;
else
- {
- /* first search (depth 0) */
- match_num = getAllMatches(ctx, ip, ip, iend, maxSearches, mls, matches, best_mlen);
- }
+ match_num = getAllMatches(ctx, ip, ip, iend, maxSearches, mls, matches, best_mlen); /* first search (depth 0) */
ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos);
if (!last_pos && !match_num) { ip++; continue; }
if (cur == last_pos) break;
- if (inr > ilimit)
+ if (inr > ilimit) // last match must start at a minimum distance of 8 from oend
continue;
mlen = opt[cur].mlen;
-#if 1
if (!opt[cur].off && opt[cur].mlen != 1) {
mlen = ZSTD_count(inr, inr - opt[cur].rep2, iend); // check rep2
ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), opt[cur].rep2, mlen);
mlen = ZSTD_count(inr, inr - opt[cur].rep, iend); // check rep
ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), opt[cur].rep, mlen);
}
-#else
- mlen = ZSTD_count(inr, inr - opt[cur].rep, iend); // check rep
-#endif
best_mlen = 0;
ZSTD_LOG_TRY_PRICE("%d: TRY8 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur2 + mlen, opt[cur2 + mlen].price);
if (cur2 + mlen > last_pos || (price < opt[cur2 + mlen].price))
- {
SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
- // opt[cur2 + mlen].rep = matches[i].off; // update reps
- // opt[cur2 + mlen].rep2 = opt[cur2].rep;
- }
mlen++;
}
// printf("orig="); print_hex_text(ip, mlen, 0);
// printf("match="); print_hex_text(ip-offset, mlen, 0);
-#if 1
+#if 0 // for debugging
size_t ml2;
if (offset)
ml2 = ZSTD_count(ip, ip-offset, iend);
projects / thirdparty / zstd.git / commitdiff
