U32 litlen;
U32 rep;
U32 rep2;
-} ZSTD_optimal_t;
+} ZSTD_optimal_t;
#define ZSTD_OPT_DEBUG 0 // 1 = tableID=0; 5 = check encoded sequences
#if 1
#define ZSTD_LOG_PARSER(fmt, args...) ;// printf(fmt, ##args)
#define ZSTD_LOG_PRICE(fmt, args...) ;//printf(fmt, ##args)
- #define ZSTD_LOG_ENCODE(fmt, args...) ;//printf(fmt, ##args)
+ #define ZSTD_LOG_ENCODE(fmt, args...) ;//printf(fmt, ##args)
#else
#define ZSTD_LOG_PARSER(fmt, args...) printf(fmt, ##args)
#define ZSTD_LOG_PRICE(fmt, args...) printf(fmt, ##args)
- #define ZSTD_LOG_ENCODE(fmt, args...) printf(fmt, ##args)
+ #define ZSTD_LOG_ENCODE(fmt, args...) printf(fmt, ##args)
#endif
#define ZSTD_LOG_TRY_PRICE(fmt, args...) ;//printf(fmt, ##args)
#define ZSTD_FREQ_THRESHOLD (256)
-// log2_32 is from http://stackoverflow.com/questions/11376288/fast-computing-of-log2-for-64-bit-integers
-const U32 tab32[32] = {
- 0, 9, 1, 10, 13, 21, 2, 29,
- 11, 14, 16, 18, 22, 25, 3, 30,
- 8, 12, 20, 28, 15, 17, 24, 7,
- 19, 27, 23, 6, 26, 5, 4, 31};
-
-U32 log2_32 (U32 value)
-{
- value |= value >> 1;
- value |= value >> 2;
- value |= value >> 4;
- value |= value >> 8;
- value |= value >> 16;
- return tab32[(U32)(value*0x07C4ACDD) >> 27];
-}
-
FORCE_INLINE U32 ZSTD_getLiteralPriceReal(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals)
{
U32 price = 0;
if (litLength > 0) {
+ U32 freq, u;
/* literals */
- for (U32 i=0; i < litLength; i++)
- price += log2_32(seqStorePtr->litSum) - log2_32(seqStorePtr->litFreq[literals[i]]);
+ for (u=0; u < litLength; u++)
+ price += ZSTD_highbit(seqStorePtr->litSum) - ZSTD_highbit(seqStorePtr->litFreq[literals[u]]);
/* literal Length */
- U32 freq;
- if (litLength >= MaxLL) {
- freq = seqStorePtr->litLengthFreq[MaxLL];
- if (litLength<255 + MaxLL) {
- price += 8;
- } else {
- price += 8;
- if (litLength < (1<<15)) price += 16; else price += 24;
- } }
- else
- freq = seqStorePtr->litLengthFreq[litLength];
- price += log2_32(seqStorePtr->litLengthSum) - log2_32(freq);
+ price += ((litLength >= MaxLL)*8) + ((litLength >= 255+MaxLL)*16) + ((litLength>=(1<<15))*8);
+ if (litLength >= MaxLL) litLength = MaxLL;
+ freq = seqStorePtr->litLengthFreq[litLength];
+ price += ZSTD_highbit(seqStorePtr->litLengthSum) - ZSTD_highbit(freq);
+ /* note : seems that using matchLengthSum (<litLengthSum) improves compression ? */
}
return price + (price == 0);
FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals)
{
- if (seqStorePtr->litSum > ZSTD_FREQ_THRESHOLD)
+ if (seqStorePtr->litSum > ZSTD_FREQ_THRESHOLD)
return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals);
-
+ /* backup eval */
return 1 + (litLength<<3);
}
FORCE_INLINE U32 ZSTD_getMatchPriceReal(seqStore_t* seqStorePtr, U32 offset, U32 matchLength)
{
- /* match offset */
- BYTE offCode = (BYTE)ZSTD_highbit(offset) + 1;
- if (offset==0)
- offCode = 0;
- U32 price = log2_32(seqStorePtr->offCodeSum) - log2_32(seqStorePtr->offCodeFreq[offCode]);
+ U32 freq;
+
+ /* offset */
+ BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset) + 1 : 0;
+ U32 price = ZSTD_highbit(seqStorePtr->offCodeSum) - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]);
price += offCode;
/* match Length */
- U32 freq;
- if (matchLength >= MaxML) {
- freq = seqStorePtr->matchLengthFreq[MaxML];
- if (matchLength < 255+MaxML) {
- price += 8;
- } else {
- price += 8;
- if (matchLength < (1<<15)) price += 16; else price += 24;
- } }
- else freq = seqStorePtr->matchLengthFreq[matchLength];
- price += log2_32(seqStorePtr->matchLengthSum) - log2_32(freq);
+ price += ((matchLength >= MaxML)*8) + ((matchLength >= 255+MaxML)*16) + ((matchLength>=(1<<15))*8);
+ if (matchLength >= MaxML) matchLength = MaxML;
+ freq = seqStorePtr->matchLengthFreq[matchLength];
+ price += ZSTD_highbit(seqStorePtr->matchLengthSum) - ZSTD_highbit(freq);
return price;
}
{
if (seqStorePtr->litSum > ZSTD_FREQ_THRESHOLD)
return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals) + ZSTD_getMatchPriceReal(seqStorePtr, offset, matchLength);
-
+
return (litLength<<3) + ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
}
MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
{
+ U32 u;
#if 0
static const BYTE* g_start = NULL;
if (g_start==NULL) g_start = literals;
#endif
/* literals */
seqStorePtr->litSum += litLength;
- for (U32 i=0; i < litLength; i++)
- seqStorePtr->litFreq[literals[i]]++;
-
+ for (u=0; u < litLength; u++)
+ seqStorePtr->litFreq[literals[u]]++;
+
/* literal Length */
seqStorePtr->litLengthSum++;
if (litLength >= MaxLL)
seqStorePtr->litLengthFreq[MaxLL]++;
- else
+ else
seqStorePtr->litLengthFreq[litLength]++;
/* match offset */
seqStorePtr->matchLengthSum++;
if (matchLength >= MaxML)
seqStorePtr->matchLengthFreq[MaxML]++;
- else
+ else
seqStorePtr->matchLengthFreq[matchLength]++;
}
U32 matchEndIdx = current+8;
U32 dummy32; /* to be nullified at the end */
U32 mnum = 0;
-
- bestLength = 0;
+
+ bestLength = MINMATCH-1;
hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow)) {
}
if (matchLength > bestLength) {
- if (matchLength > matchEndIdx - matchIndex)
- matchEndIdx = matchIndex + (U32)matchLength;
- {
- if (matchLength >= MINMATCH) {
- bestLength = matchLength;
- matches[mnum].off = current - matchIndex;
- matches[mnum].len = (U32)matchLength;
- matches[mnum].back = 0;
- mnum++;
- }
- if (matchLength > ZSTD_OPT_NUM) break;
- }
+ if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength;
+ bestLength = matchLength;
+ matches[mnum].off = current - matchIndex;
+ matches[mnum].len = (U32)matchLength;
+ matches[mnum].back = 0;
+ mnum++;
+ if (matchLength > ZSTD_OPT_NUM) break;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */
}
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
largerPtr = nextPtr;
matchIndex = nextPtr[0];
- }
- }
+ } }
*smallerPtr = *largerPtr = 0;
const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit,
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml)
{
- if (iLowLimit) {}; // skip warnings
-
+ (void)iLowLimit; /* unused */
switch(matchLengthSearch)
{
default :
/** Tree updater, providing best match */
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
-size_t ZSTD_BtGetAllMatches_extDict (
+U32 ZSTD_BtGetAllMatches_extDict (
ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iLimit,
const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, U32 minml)
}
/* save best solution */
- if (currentMl > minml) {
- minml = currentMl;
+ if (currentMl > minml) {
+ minml = currentMl;
matches[mnum].off = current - matchIndex;
matches[mnum].len = (U32)currentMl;
matches[mnum].back = back;
mnum++;
if (currentMl > ZSTD_OPT_NUM) break;
- if (ip+currentMl == iHighLimit) break; /* best possible, and avoid read overflow*/
+ if (ip+currentMl == iHighLimit) break; /* best possible, and avoid read overflow*/
}
if (matchIndex <= minChain) break;
ZSTD_optimal_t opt[ZSTD_OPT_NUM+4];
ZSTD_match_t matches[ZSTD_OPT_NUM+1];
- const uint8_t *inr;
+ const BYTE* inr;
U32 skip_num, cur, cur2, match_num, last_pos, litlen, price;
const U32 sufficient_len = ctx->params.sufficientLength;
- const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt);
+ const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt);
/* init */
/* Match Loop */
while (ip < ilimit) {
+ U32 u;
U32 mlen=0;
U32 best_mlen=0;
U32 best_off=0;
if (MEM_read32(ip+1) == MEM_read32(ip+1 - rep_1)) {
/* repcode : we take it */
mlen = (U32)ZSTD_count(ip+1+MINMATCH, ip+1+MINMATCH-rep_1, iend) + MINMATCH;
-
+
ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep_1, (int)mlen);
if (depth==0 || mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
ip+=1; best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1;
best_mlen = (last_pos) ? last_pos : MINMATCH;
-
+
if (faster_get_matches && last_pos)
match_num = 0;
else
opt[0].rep2 = rep_2;
opt[0].mlen = 1;
- if (match_num && matches[match_num-1].len > sufficient_len)
- {
+ if (match_num && matches[match_num-1].len > sufficient_len) {
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
cur = 0;
}
// set prices using matches at position = 0
- for (U32 i = 0; i < match_num; i++)
- {
- mlen = (i>0) ? matches[i-1].len+1 : best_mlen;
- best_mlen = (matches[i].len < ZSTD_OPT_NUM) ? matches[i].len : ZSTD_OPT_NUM;
+ for (u = 0; u < match_num; u++) {
+ mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
+ best_mlen = (matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM;
ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos);
litlen = opt[0].litlen;
while (mlen <= best_mlen)
{
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[i].off, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
if (mlen > last_pos || price < opt[mlen].price)
- SET_PRICE(mlen, mlen, matches[i].off, litlen, price);
+ SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
mlen++;
}
}
- if (last_pos < MINMATCH) {
+ if (last_pos < MINMATCH) {
// ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
- ip++; continue;
+ ip++; continue;
}
// check further positions
for (skip_num = 0, cur = 1; cur <= last_pos; cur++)
- {
+ {
+ size_t cur_rep;
inr = ip + cur;
- if (opt[cur-1].mlen == 1)
- {
+ if (opt[cur-1].mlen == 1) {
litlen = opt[cur-1].litlen + 1;
- if (cur > litlen)
- {
+ if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen);
ZSTD_LOG_TRY_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-base), cur - litlen, opt[cur - litlen].price, price, cur, litlen);
- }
- else
+ } else
price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
- }
- else
- {
+ } else {
litlen = 1;
- price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
+ price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
ZSTD_LOG_TRY_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-base), price, cur, litlen, (int)ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1));
}
-
+
ZSTD_LOG_TRY_PRICE("%d: TRY4 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur, opt[cur].price);
if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen)))
continue;
mlen = opt[cur].mlen;
-
- if (opt[cur-mlen].off)
- {
+
+ if (opt[cur-mlen].off) {
opt[cur].rep2 = opt[cur-mlen].rep;
opt[cur].rep = opt[cur-mlen].off;
ZSTD_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2);
- }
- else
- {
- if (cur!=mlen && opt[cur-mlen].litlen == 0)
- {
+ } else {
+ if (cur!=mlen && opt[cur-mlen].litlen == 0) {
opt[cur].rep2 = opt[cur-mlen].rep;
opt[cur].rep = opt[cur-mlen].rep2;
ZSTD_LOG_PARSER("%d: COPYREP2 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2);
- }
- else
- {
+ } else {
opt[cur].rep2 = opt[cur-mlen].rep2;
opt[cur].rep = opt[cur-mlen].rep;
ZSTD_LOG_PARSER("%d: COPYREP3 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2);
- }
- }
-
- ZSTD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2);
-
+ } }
+ ZSTD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2);
- size_t cur_rep;
best_mlen = 0;
if (!opt[cur].off && opt[cur].mlen != 1) {
cur_rep = opt[cur].rep2;
- ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);
- }
- else {
+ ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);
+ } else {
cur_rep = opt[cur].rep;
- ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);
+ ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);
}
- if (MEM_read32(inr) == MEM_read32(inr - cur_rep)) // check rep
- {
- mlen = (U32)ZSTD_count(inr+MINMATCH, inr+MINMATCH - cur_rep, iend) + MINMATCH;
+ if (MEM_read32(inr) == MEM_read32(inr - cur_rep)) { // check rep
+ mlen = (U32)ZSTD_count(inr+MINMATCH, inr+MINMATCH - cur_rep, iend) + MINMATCH;
ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), mlen, 0, opt[cur].rep, cur, opt[cur].off);
- if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM)
- {
+ if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen;
best_off = 0;
ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
goto _storeSequence;
}
- if (opt[cur].mlen == 1)
- {
+ if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
- if (cur > litlen)
- {
+ if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, 0, mlen - MINMATCH);
ZSTD_LOG_TRY_PRICE("%d: TRY5 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-base), cur - litlen, opt[cur - litlen].price, price, cur, litlen);
- }
- else
+ } else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, 0, mlen - MINMATCH);
- }
- else
- {
+ } else {
litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, 0, mlen - MINMATCH);
ZSTD_LOG_TRY_PRICE("%d: TRY7 price=%d cur=%d litlen=0 getprice=%d\n", (int)(inr-base), price, cur, (int)ZSTD_getPrice(seqStorePtr, 0, NULL, 0, mlen - MINMATCH));
ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), mlen, 0, price, litlen, cur - litlen, opt[cur - litlen].price);
- do
- {
+ do {
if (cur + mlen > last_pos || price <= opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH
SET_PRICE(cur + mlen, mlen, 0, litlen, price);
mlen--;
- }
- while (mlen >= MINMATCH);
+ } while (mlen >= MINMATCH);
}
-
- if (faster_get_matches && skip_num > 0)
- {
- skip_num--;
- continue;
- }
+ if (faster_get_matches && skip_num > 0) { skip_num--; continue; }
- best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH;
+ best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH;
- match_num = getAllMatches(ctx, inr, ip, iend, maxSearches, mls, matches, best_mlen);
+ match_num = getAllMatches(ctx, inr, ip, iend, maxSearches, mls, matches, best_mlen);
ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num);
- if (match_num > 0 && matches[match_num-1].len > sufficient_len)
- {
+ if (match_num > 0 && matches[match_num-1].len > sufficient_len) {
cur -= matches[match_num-1].back;
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
}
- // set prices using matches at position = cur
- for (U32 i = 0; i < match_num; i++)
- {
- mlen = (i>0) ? matches[i-1].len+1 : best_mlen;
- cur2 = cur - matches[i].back;
- best_mlen = (cur2 + matches[i].len < ZSTD_OPT_NUM) ? matches[i].len : ZSTD_OPT_NUM - cur2;
+ /* set prices using matches at position = cur */
+ for (u = 0; u < match_num; u++) {
+ mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
+ cur2 = cur - matches[u].back;
+ best_mlen = (cur2 + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur2;
- ZSTD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos);
- if (mlen < matches[i].back + 1)
- mlen = matches[i].back + 1;
+ ZSTD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, cur2, matches[u].len, matches[i].off, best_mlen, last_pos);
+ if (mlen < matches[u].back + 1)
+ mlen = matches[u].back + 1;
- while (mlen <= best_mlen)
- {
- if (opt[cur2].mlen == 1)
- {
+ while (mlen <= best_mlen) {
+ if (opt[cur2].mlen == 1) {
litlen = opt[cur2].litlen;
if (cur2 > litlen)
- price = opt[cur2 - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur2-litlen, matches[i].off, mlen - MINMATCH);
+ price = opt[cur2 - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur2-litlen, matches[u].off, mlen - MINMATCH);
else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[i].off, mlen - MINMATCH);
- }
- else
- {
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
+ } else {
litlen = 0;
- price = opt[cur2].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[i].off, mlen - MINMATCH);
+ price = opt[cur2].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH);
}
- ZSTD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price);
+ ZSTD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), matches[u].back, mlen, best_mlen, matches[u].off, price, litlen, cur - litlen, opt[cur - litlen].price);
ZSTD_LOG_TRY_PRICE("%d: TRY8 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur2 + mlen, opt[cur2 + mlen].price);
if (cur2 + mlen > last_pos || (price < opt[cur2 + mlen].price))
- SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
+ SET_PRICE(cur2 + mlen, mlen, matches[u].off, litlen, price);
mlen++;
- }
- }
+ } }
} // for (skip_num = 0, cur = 1; cur <= last_pos; cur++)
best_mlen = opt[last_pos].mlen;
best_off = opt[last_pos].off;
cur = last_pos - best_mlen;
- // printf("%d: start=%d best_mlen=%d best_off=%d cur=%d\n", (int)(ip - base), (int)(start - ip), (int)best_mlen, (int)best_off, cur);
+ // printf("%d: start=%d best_mlen=%d best_off=%d cur=%d\n", (int)(ip - base), (int)(start - ip), (int)best_mlen, (int)best_off, cur);
/* store sequence */
_storeSequence: // cur, last_pos, best_mlen, best_off have to be set
- for (U32 i = 1; i <= last_pos; i++)
- ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep, opt[i].rep2);
- ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep);
+ for (u = 1; u <= last_pos; u++)
+ ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2);
+ ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep);
opt[0].mlen = 1;
U32 offset;
- while (1)
- {
+ while (1) {
mlen = opt[cur].mlen;
ZSTD_LOG_PARSER("%d: cur=%d mlen=%d\n", (int)(ip-base), cur, mlen);
offset = opt[cur].off;
- opt[cur].mlen = best_mlen;
+ opt[cur].mlen = best_mlen;
opt[cur].off = best_off;
best_mlen = mlen;
- best_off = offset;
+ best_off = offset;
if (mlen > cur)
break;
cur -= mlen;
}
-
- for (U32 i = 0; i <= last_pos;)
- {
- ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep, opt[i].rep2);
- i += opt[i].mlen;
+
+ for (u = 0; u <= last_pos;) {
+ ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2);
+ u += opt[u].mlen;
}
cur = 0;
}
else
{
- if (litLength == 0)
+ if (litLength == 0)
{
best_off = rep_2;
rep_2 = rep_1;
}
ZSTD_LOG_ENCODE("%d/%d: ENCODE2 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2);
-
+
#if ZSTD_OPT_DEBUG >= 5
int ml2;
if (offset)
anchor += best_mlen+MINMATCH;
ip = anchor;
continue; // faster when present ... (?)
- }
+ }
}
/* Last Literals */
ZSTD_optimal_t opt[ZSTD_OPT_NUM+4];
ZSTD_match_t matches[ZSTD_OPT_NUM+1];
- const uint8_t *inr;
+ const BYTE* inr;
U32 skip_num, cur, cur2, match_num, last_pos, litlen, price;
const U32 sufficient_len = ctx->params.sufficientLength;
- const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt);
+ const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt);
/* init */
/* Match Loop */
while (ip < ilimit) {
+ U32 u;
+ U32 offset;
U32 mlen=0;
U32 best_mlen=0;
U32 best_off=0;
/* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
mlen = (U32)ZSTD_count_2segments(ip+1+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH;
-
+
ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep_1, (int)mlen);
if (depth==0 || mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
ip+=1; best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1;
}
litlen = opt[0].litlen + 1;
- do
- {
+ do {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, 0, mlen - MINMATCH);
if (mlen + 1 > last_pos || price < opt[mlen + 1].price)
SET_PRICE(mlen + 1, mlen, 0, litlen, price);
mlen--;
- }
- while (mlen >= MINMATCH);
- }
- }
+ } while (mlen >= MINMATCH);
+ } }
best_mlen = (last_pos) ? last_pos : MINMATCH;
-
+
if (faster_get_matches && last_pos)
match_num = 0;
else
opt[0].rep2 = rep_2;
opt[0].mlen = 1;
- if (match_num && matches[match_num-1].len > sufficient_len)
- {
+ if (match_num && matches[match_num-1].len > sufficient_len) {
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
cur = 0;
}
// set prices using matches at position = 0
- for (U32 i = 0; i < match_num; i++)
- {
- mlen = (i>0) ? matches[i-1].len+1 : best_mlen;
- best_mlen = (matches[i].len < ZSTD_OPT_NUM) ? matches[i].len : ZSTD_OPT_NUM;
- ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos);
+ for (u = 0; u < match_num; u++) {
+ mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
+ best_mlen = (matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM;
+ ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
litlen = opt[0].litlen;
- while (mlen <= best_mlen)
- {
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[i].off, mlen - MINMATCH);
+ while (mlen <= best_mlen) {
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
if (mlen > last_pos || price < opt[mlen].price)
- SET_PRICE(mlen, mlen, matches[i].off, litlen, price);
+ SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
mlen++;
- }
- }
+ } }
- if (last_pos < MINMATCH) {
- // ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
- ip++; continue;
+ if (last_pos < MINMATCH) {
+ // ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
+ ip++; continue;
}
// check further positions
for (skip_num = 0, cur = 1; cur <= last_pos; cur++)
- {
+ {
+ size_t cur_rep;
inr = ip + cur;
- if (opt[cur-1].mlen == 1)
- {
+ if (opt[cur-1].mlen == 1) {
litlen = opt[cur-1].litlen + 1;
- if (cur > litlen)
- {
+ if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen);
ZSTD_LOG_TRY_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-base), cur - litlen, opt[cur - litlen].price, price, cur, litlen);
- }
- else
+ } else
price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
- }
- else
- {
+ } else {
litlen = 1;
- price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
+ price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
ZSTD_LOG_TRY_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-base), price, cur, litlen, (int)ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1));
}
-
+
ZSTD_LOG_TRY_PRICE("%d: TRY4 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur, opt[cur].price);
if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen)))
continue;
mlen = opt[cur].mlen;
-
- if (opt[cur-mlen].off)
- {
+
+ if (opt[cur-mlen].off) {
opt[cur].rep2 = opt[cur-mlen].rep;
opt[cur].rep = opt[cur-mlen].off;
ZSTD_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2);
- }
- else
- {
- if (cur!=mlen && opt[cur-mlen].litlen == 0)
- {
+ } else {
+ if (cur!=mlen && opt[cur-mlen].litlen == 0) {
opt[cur].rep2 = opt[cur-mlen].rep;
opt[cur].rep = opt[cur-mlen].rep2;
ZSTD_LOG_PARSER("%d: COPYREP2 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2);
- }
- else
- {
+ } else {
opt[cur].rep2 = opt[cur-mlen].rep2;
opt[cur].rep = opt[cur-mlen].rep;
ZSTD_LOG_PARSER("%d: COPYREP3 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2);
- }
- }
+ } }
- ZSTD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2);
+ ZSTD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2);
- size_t cur_rep;
best_mlen = 0;
if (!opt[cur].off && opt[cur].mlen != 1) {
cur_rep = opt[cur].rep2;
- ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);
+ ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);
}
else {
cur_rep = opt[cur].rep;
- ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);
+ ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);
}
const U32 repIndex = (U32)(current+cur - cur_rep);
if (faster_get_matches && skip_num > 0)
{
- skip_num--;
+ skip_num--;
continue;
}
- best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH;
+ best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH;
- match_num = getAllMatches(ctx, inr, ip, iend, maxSearches, mls, matches, best_mlen);
+ match_num = getAllMatches(ctx, inr, ip, iend, maxSearches, mls, matches, best_mlen);
ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num);
- if (match_num > 0 && matches[match_num-1].len > sufficient_len)
- {
+ if (match_num > 0 && matches[match_num-1].len > sufficient_len) {
cur -= matches[match_num-1].back;
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
// set prices using matches at position = cur
- for (U32 i = 0; i < match_num; i++)
- {
- mlen = (i>0) ? matches[i-1].len+1 : best_mlen;
- cur2 = cur - matches[i].back;
- best_mlen = (cur2 + matches[i].len < ZSTD_OPT_NUM) ? matches[i].len : ZSTD_OPT_NUM - cur2;
+ for (u = 0; u < match_num; u++) {
+ mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
+ cur2 = cur - matches[u].back;
+ best_mlen = (cur2 + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur2;
- ZSTD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos);
- if (mlen < matches[i].back + 1)
- mlen = matches[i].back + 1;
+ ZSTD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, cur2, matches[u].len, matches[u].off, best_mlen, last_pos);
+ if (mlen < matches[u].back + 1)
+ mlen = matches[u].back + 1;
- while (mlen <= best_mlen)
- {
- if (opt[cur2].mlen == 1)
- {
+ while (mlen <= best_mlen) {
+ if (opt[cur2].mlen == 1) {
litlen = opt[cur2].litlen;
if (cur2 > litlen)
- price = opt[cur2 - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur2-litlen, matches[i].off, mlen - MINMATCH);
+ price = opt[cur2 - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur2-litlen, matches[u].off, mlen - MINMATCH);
else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[i].off, mlen - MINMATCH);
- }
- else
- {
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
+ } else {
litlen = 0;
- price = opt[cur2].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[i].off, mlen - MINMATCH);
+ price = opt[cur2].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH);
}
- ZSTD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price);
+ ZSTD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), matches[u].back, mlen, best_mlen, matches[u].off, price, litlen, cur - litlen, opt[cur - litlen].price);
ZSTD_LOG_TRY_PRICE("%d: TRY8 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur2 + mlen, opt[cur2 + mlen].price);
if (cur2 + mlen > last_pos || (price < opt[cur2 + mlen].price))
- SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
+ SET_PRICE(cur2 + mlen, mlen, matches[u].off, litlen, price);
mlen++;
- }
- }
+ } }
} // for (skip_num = 0, cur = 1; cur <= last_pos; cur++)
best_mlen = opt[last_pos].mlen;
best_off = opt[last_pos].off;
cur = last_pos - best_mlen;
- // printf("%d: start=%d best_mlen=%d best_off=%d cur=%d\n", (int)(ip - base), (int)(start - ip), (int)best_mlen, (int)best_off, cur);
+ // printf("%d: start=%d best_mlen=%d best_off=%d cur=%d\n", (int)(ip - base), (int)(start - ip), (int)best_mlen, (int)best_off, cur);
/* store sequence */
_storeSequence: // cur, last_pos, best_mlen, best_off have to be set
- for (U32 i = 1; i <= last_pos; i++)
- ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep, opt[i].rep2);
- ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep);
+ for (u = 1; u <= last_pos; u++)
+ ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u]litlen, opt[u].rep, opt[u].rep2);
+ ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep);
opt[0].mlen = 1;
- U32 offset;
- while (1)
- {
+ while (1) {
mlen = opt[cur].mlen;
ZSTD_LOG_PARSER("%d: cur=%d mlen=%d\n", (int)(ip-base), cur, mlen);
offset = opt[cur].off;
- opt[cur].mlen = best_mlen;
+ opt[cur].mlen = best_mlen;
opt[cur].off = best_off;
best_mlen = mlen;
- best_off = offset;
+ best_off = offset;
if (mlen > cur)
break;
cur -= mlen;
}
-
- for (U32 i = 0; i <= last_pos;)
- {
- ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep, opt[i].rep2);
- i += opt[i].mlen;
+
+ for (u = 0; u <= last_pos;) {
+ ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2);
+ u += opt[u].mlen;
}
cur = 0;
- while (cur < last_pos)
- {
+ while (cur < last_pos) {
+ U32 litLength;
ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2);
mlen = opt[cur].mlen;
if (mlen == 1) { ip++; cur++; continue; }
offset = opt[cur].off;
cur += mlen;
-
- U32 litLength = (U32)(ip - anchor);
+ litLength = (U32)(ip - anchor);
ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2);
- if (offset)
- {
+ if (offset) {
rep_2 = rep_1;
rep_1 = offset;
- }
- else
- {
- if (litLength == 0)
- {
+ } else {
+ if (litLength == 0) {
best_off = rep_2;
rep_2 = rep_1;
rep_1 = best_off;
- }
- }
+ } }
ZSTD_LOG_ENCODE("%d/%d: ENCODE2 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2);
-
+
#if ZSTD_OPT_DEBUG >= 5
int ml2;
if (offset)
}
- // check immediate repcode
+ /* check immediate repcode */
while (anchor <= ilimit) {
const U32 repIndex = (U32)((anchor-base) - rep_2);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
- if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
- if (MEM_read32(anchor) == MEM_read32(repMatch)) {
- /* repcode detected we should take it */
+ if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
+ && (MEM_read32(anchor) == MEM_read32(repMatch)) ) {
+ /* repcode detected, let's take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
mlen = (U32)ZSTD_count_2segments(anchor+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH;
offset = rep_2; rep_2 = rep_1; rep_1 = offset; /* swap offset history */
continue; /* faster when present ... (?) */
}
break;
- }
- }
+ } }
/* Last Literals */
{
/*
- paramgrill.c - parameter tester for zstd_hc
- Copyright (C) Yann Collet 2015
+ paramgrill.c - parameter tester for zstd
+ Copyright (C) Yann Collet 2015-2016
GPL v2 License
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- - zstd source repository : https://github.com/Cyan4973/zstd
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+ - zstd homepage : http://www.zstd.net/
*/
-/**************************************
+/*-************************************
* Compiler Options
**************************************/
/* Disable some Visual warning messages */
#endif
-/**************************************
-* Includes
+/*-************************************
+* Dependencies
**************************************/
#include <stdlib.h> /* malloc */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include "xxhash.h"
-/**************************************
+/*-************************************
* Compiler Options
**************************************/
/* S_ISREG & gettimeofday() are not supported by MSVC */
#endif
-/**************************************
+/*-************************************
* Constants
**************************************/
#define PROGRAM_DESCRIPTION "ZSTD_HC parameters tester"
#define NBLOOPS 2
#define TIMELOOP 2000
+#define NB_LEVELS_TRACKED 30
+
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
#define DEFAULT_CHUNKSIZE (4<<20)
static const int g_maxNbVariations = 64;
-/**************************************
+/*-************************************
* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-/**************************************
+/*-************************************
* Benchmark Parameters
**************************************/
static U32 g_nbIterations = NBLOOPS;
static U32 g_singleRun = 0;
static U32 g_target = 0;
static U32 g_noSeed = 0;
-static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
+static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, 0, ZSTD_greedy };
void BMK_SetNbIterations(int nbLoops)
{
}
-/*********************************************************
+/*-*******************************************************
* Private functions
*********************************************************/
if (requiredMem > maxMemory) requiredMem = maxMemory;
requiredMem += 2*step;
- while (!testmem)
- {
+ while (!testmem) {
requiredMem -= step;
testmem = (BYTE*) malloc ((size_t)requiredMem);
}
}
-/*********************************************************
+/*-*******************************************************
* Bench functions
*********************************************************/
typedef struct {
U32 Hlog = params.hashLog;
U32 Slog = params.searchLog;
U32 Slength = params.searchLength;
+ U32 Tlength = params.sufficientLength;
ZSTD_strategy strat = params.strategy;
char name[30] = { 0 };
U64 crcOrig;
/* Memory allocation & restrictions */
- snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%1u", Wlog, Clog, Hlog, Slog, Slength, strat);
- if (!compressedBuffer || !resultBuffer || !blockTable)
- {
+ snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%03uS%1u", Wlog, Clog, Hlog, Slog, Slength, Tlength, strat);
+ if (!compressedBuffer || !resultBuffer || !blockTable) {
DISPLAY("\nError: not enough memory!\n");
free(compressedBuffer);
free(resultBuffer);
const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)resultBuffer;
- for (i=0; i<nbBlocks; i++)
- {
+ for (i=0; i<nbBlocks; i++) {
size_t thisBlockSize = MIN(remaining, blockSize);
blockTable[i].srcPtr = srcPtr;
blockTable[i].cPtr = cPtr;
cPtr += blockTable[i].cRoom;
resPtr += thisBlockSize;
remaining -= thisBlockSize;
- }
- }
+ } }
/* warmimg up memory */
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.10, 1);
const int startTime =BMK_GetMilliStart();
DISPLAY("\r%79s\r", "");
- for (loopNb = 1; loopNb <= g_nbIterations; loopNb++)
- {
+ for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
int nbLoops;
int milliTime;
U32 blockNb;
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
- while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
- {
+ while (BMK_GetMilliSpan(milliTime) < TIMELOOP) {
for (blockNb=0; blockNb<nbBlocks; blockNb++)
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
- for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++)
- {
+ for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++) {
for (blockNb=0; blockNb<nbBlocks; blockNb++)
blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
/* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0);
- if (crcOrig!=crcCheck)
- {
+ if (crcOrig!=crcCheck) {
unsigned u;
unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck);
- for (u=0; u<srcSize; u++)
- {
- if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u])
- {
+ for (u=0; u<srcSize; u++) {
+ if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) {
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
- }
- }
+ } }
break;
}
#endif
- }
- }
+ } }
/* End cleaning */
DISPLAY("\r");
"ZSTD_greedy ",
"ZSTD_lazy ",
"ZSTD_lazy2 ",
- "ZSTD_btlazy2" };
+ "ZSTD_btlazy2",
+ "ZSTD_opt ",
+ "ZSTD_opt_bt " };
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_parameters params, size_t srcSize)
{
DISPLAY("\r%79s\r", "");
- fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
+ fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
0, params.windowLog, params.contentLog, params.hashLog, params.searchLog, params.searchLength,
- g_stratName[(U32)(params.strategy)]);
+ params.sufficientLength, g_stratName[(U32)(params.strategy)]);
fprintf(f,
"/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
cLevel, (double)srcSize / result.cSize, (double)result.cSpeed / 1000., (double)result.dSpeed / 1000.);
}
-static U32 g_cSpeedTarget[ZSTD_MAX_CLEVEL+1] = { 0 };
+static U32 g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0 }; /* NB_LEVELS_TRACKED : to check */
typedef struct {
BMK_result_t result;
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
{
- int cLevel;
+ unsigned cLevel;
fprintf(f, "\n /* Proposed configurations : */ \n");
- fprintf(f, "#define ZSTD_MAX_CLEVEL %2u \n", ZSTD_MAX_CLEVEL);
- fprintf(f, "static const ZSTD_parameters ZSTD_defaultParameters[ZSTD_MAX_CLEVEL+1] = {\n");
fprintf(f, " /* l, W, C, H, S, L, strat */ \n");
- for (cLevel=0; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
+ for (cLevel=0; cLevel <= ZSTD_maxCLevel(); cLevel++)
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
}
{
BMK_result_t testResult;
int better = 0;
- int cLevel;
+ unsigned cLevel;
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
- for (cLevel = 1; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
- {
+ for (cLevel = 1; cLevel <= ZSTD_maxCLevel(); cLevel++) {
if (testResult.cSpeed < g_cSpeedTarget[cLevel])
continue; /* not fast enough for this level */
- if (winners[cLevel].result.cSize==0)
- {
+ if (winners[cLevel].result.cSize==0) {
/* first solution for this cLevel */
winners[cLevel].result = testResult;
winners[cLevel].params = params;
continue;
}
- if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) )
- {
+ if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
/* Validate solution is "good enough" */
double W_ratio = (double)srcSize / testResult.cSize;
double O_ratio = (double)srcSize / winners[cLevel].result.cSize;
double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed);
- if (W_DMemUsed_note < O_DMemUsed_note)
- {
+ if (W_DMemUsed_note < O_DMemUsed_note) {
/* uses too much Decompression memory for too little benefit */
if (W_ratio > O_ratio)
DISPLAY ("Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n",
O_ratio, (double)(O_DMemUsed) / 1024 / 1024, cLevel);
continue;
}
- if (W_CMemUsed_note < O_CMemUsed_note)
- {
+ if (W_CMemUsed_note < O_CMemUsed_note) {
/* uses too much memory for compression for too little benefit */
if (W_ratio > O_ratio)
DISPLAY ("Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n",
O_ratio, (double)(O_CMemUsed) / 1024 / 1024, cLevel);
continue;
}
- if (W_CSpeed_note < O_CSpeed_note )
- {
+ if (W_CSpeed_note < O_CSpeed_note ) {
/* too large compression speed difference for the compression benefit */
if (W_ratio > O_ratio)
DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
O_ratio, (double)(winners[cLevel].result.cSpeed) / 1000., cLevel);
continue;
}
- if (W_DSpeed_note < O_DSpeed_note )
- {
+ if (W_DSpeed_note < O_DSpeed_note ) {
/* too large decompression speed difference for the compression benefit */
if (W_ratio > O_ratio)
DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
better = 1;
- }
-
- }
+ } }
return better;
}
{
g_params = params;
if (params.strategy == ZSTD_fast)
- {
- g_params.contentLog = 0;
- g_params.searchLog = 0;
- }
+ g_params.contentLog = 0, g_params.searchLog = 0;
+ if ((params.strategy != ZSTD_opt) && (params.strategy != ZSTD_opt_bt))
+ g_params.sufficientLength = 0;
return &g_params;
}
static void paramVariation(ZSTD_parameters* p)
{
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
- for (; nbChanges; nbChanges--)
- {
+ for (; nbChanges; nbChanges--) {
const U32 changeID = FUZ_rand(&g_rand) % 12;
switch(changeID)
{
p->strategy = (ZSTD_strategy)(((U32)p->strategy)+1); break;
case 11:
p->strategy = (ZSTD_strategy)(((U32)p->strategy)-1); break;
+ case 12:
+ p->sufficientLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
+ case 13:
+ p->sufficientLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
}
}
ZSTD_validateParams(p);
int nbVariations = 0;
const int startTime = BMK_GetMilliStart();
- while (BMK_GetMilliSpan(startTime) < g_maxVariationTime)
- {
+ while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) {
ZSTD_parameters p = params;
if (nbVariations++ > g_maxNbVariations) break;
static void potentialRandomParams(ZSTD_parameters* p, U32 inverseChance)
{
U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1));
- if (!chance)
- {
+ if (!chance) {
/* totally random entry */
p->contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN;
p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
p->searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
p->windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
- p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btlazy2+1));
+ p->sufficientLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
+ p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_opt_bt+1));
ZSTD_validateParams(p);
}
}
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx)
{
- U32 id = (FUZ_rand(&g_rand) % (ZSTD_MAX_CLEVEL+1));
- if ((id==0) || (winners[id].params.windowLog==0))
- {
+ U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
+ if ((id==0) || (winners[id].params.windowLog==0)) {
/* totally random entry */
ZSTD_parameters p;
potentialRandomParams(&p, 1);
{
ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_parameters params;
- winnerInfo_t winners[ZSTD_MAX_CLEVEL+1];
+ winnerInfo_t winners[NB_LEVELS_TRACKED];
int i;
+ unsigned u;
const char* rfName = "grillResults.txt";
FILE* f;
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
- if (g_singleRun)
- {
+ if (g_singleRun) {
BMK_result_t testResult;
g_params.srcSize = blockSize;
ZSTD_validateParams(&g_params);
if (g_target)
g_cSpeedTarget[1] = g_target * 1000;
- else
- {
+ else {
/* baseline config for level 1 */
BMK_result_t testResult;
params = ZSTD_getParams(1, blockSize);
}
/* establish speed objectives (relative to level 1) */
- for (i=2; i<=ZSTD_MAX_CLEVEL; i++)
- g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) >> 5;
+ for (u=2; u<=ZSTD_maxCLevel(); u++)
+ g_cSpeedTarget[u] = (g_cSpeedTarget[u-1] * 25) >> 5;
/* populate initial solution */
{
- const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL;
- for (i=1; i<=maxSeeds; i++)
- {
+ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
+ for (i=1; i<=maxSeeds; i++) {
params = ZSTD_getParams(i, blockSize);
ZSTD_validateParams(¶ms);
BMK_seed(winners, params, srcBuffer, srcSize, ctx);
/* start tests */
{
const int milliStart = BMK_GetMilliStart();
- do
- {
+ do {
BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx);
} while (BMK_GetMilliSpan(milliStart) < g_grillDuration);
}
static int benchSample(void)
{
- char* origBuff;
+ void* origBuff;
size_t benchedSize = sampleSize;
const char* name = "Sample 10MiB";
/* Allocation */
- origBuff = (char*) malloc((size_t)benchedSize);
- if(!origBuff)
- {
- DISPLAY("\nError: not enough memory!\n");
- return 12;
- }
+ origBuff = malloc(benchedSize);
+ if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; }
/* Fill buffer */
RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
int fileIdx=0;
/* Loop for each file */
- while (fileIdx<nbFiles)
- {
+ while (fileIdx<nbFiles) {
FILE* inFile;
char* inFileName;
U64 inFileSize;
/* Check file existence */
inFileName = fileNamesTable[fileIdx++];
inFile = fopen( inFileName, "rb" );
- if (inFile==NULL)
- {
+ if (inFile==NULL) {
DISPLAY( "Pb opening %s\n", inFileName);
return 11;
}
/* Memory allocation & restrictions */
inFileSize = BMK_GetFileSize(inFileName);
- benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3;
+ benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
- {
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
- }
/* Alloc */
origBuff = (char*) malloc((size_t)benchedSize);
- if(!origBuff)
- {
+ if(!origBuff) {
DISPLAY("\nError: not enough memory!\n");
fclose(inFile);
return 12;
readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile);
- if(readSize != benchedSize)
- {
+ if(readSize != benchedSize) {
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(origBuff);
return 13;
/* Check file existence */
inFile = fopen( inFileName, "rb" );
- if (inFile==NULL)
- {
+ if (inFile==NULL) {
DISPLAY( "Pb opening %s\n", inFileName);
return 11;
}
benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3;
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
- {
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
- }
/* Alloc */
origBuff = (char*) malloc((size_t)benchedSize);
- if(!origBuff)
- {
+ if(!origBuff) {
DISPLAY("\nError: not enough memory!\n");
fclose(inFile);
return 12;
readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile);
- if(readSize != benchedSize)
- {
+ if(readSize != benchedSize) {
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(origBuff);
return 13;
/* find best solution from default params */
{
- const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL;
- for (i=1; i<=maxSeeds; i++)
- {
+ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
+ for (i=1; i<=maxSeeds; i++) {
params = ZSTD_getParams(i, blockSize);
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params);
if ( (candidate.cSize < winner.result.cSize)
winner.params = params;
winner.result = candidate;
BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize);
- }
- }
+ } }
}
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
/* start tests */
{
const int milliStart = BMK_GetMilliStart();
- do
- {
+ do {
params = winner.params;
paramVariation(¶ms);
potentialRandomParams(¶ms, 16);
/* improvement found => new winner */
if ( (candidate.cSize < winner.result.cSize)
- ||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) )
- {
+ ||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) ) {
winner.params = params;
winner.result = candidate;
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
}
-
} while (BMK_GetMilliSpan(milliStart) < g_grillDuration);
}
U32 optimizer = 0;
U32 main_pause = 0;
+ /* checks */
+ if (NB_LEVELS_TRACKED <= ZSTD_maxCLevel()) {
+ DISPLAY("Error : NB_LEVELS_TRACKED <= ZSTD_maxCLevel() \n");
+ exit(1);
+ }
+
/* Welcome message */
DISPLAY(WELCOME_MESSAGE);
if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
/* Decode command (note : aggregated commands are allowed) */
- if (argument[0]=='-')
- {
+ if (argument[0]=='-') {
argument++;
- while (argument[0]!=0)
- {
+ while (argument[0]!=0) {
switch(argument[0])
{
argument++;
{
U32 proba32 = 0;
- while ((argument[0]>= '0') && (argument[0]<= '9'))
- {
+ while ((argument[0]>= '0') && (argument[0]<= '9')) {
proba32 *= 10;
proba32 += argument[0] - '0';
argument++;
g_singleRun = 1;
argument++;
g_params = ZSTD_getParams(2, g_blockSize);
- for ( ; ; )
- {
+ for ( ; ; ) {
switch(*argument)
{
case 'w':
case 't': /* strategy */
g_params.strategy = (ZSTD_strategy)0;
argument++;
- while ((*argument>= '0') && (*argument<='9'))
- {
+ while ((*argument>= '0') && (*argument<='9')) {
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy *10);
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy + *argument++ - '0');
}
case 'T':
argument++;
g_target = 0;
- while ((*argument >= '0') && (*argument <= '9'))
- {
+ while ((*argument >= '0') && (*argument <= '9')) {
g_target *= 10;
g_target += *argument - '0';
argument++;
if (filenamesStart==0)
result = benchSample();
- else
- {
+ else {
if (optimizer)
result = optimizeForSize(input_filename);
else
return result;
}
-
projects / thirdparty / zstd.git / commitdiff
