ZSTD_resetFreqs

diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c
index 601df81597ca4c745e412e8b3370a383dbc86ade..0b299349cfa52f7d2cf02935b3e99e7e7ad9265a 100644 (file)
--- a/lib/zstd_compress.c
+++ b/lib/zstd_compress.c
@@ -98,14 +98,9 @@ typedef struct {
     U32 offCodeSum;
 } seqStore_t;
 
-static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
+static void ZSTD_resetFreqs(seqStore_t* ssPtr)
 {
-    ssPtr->offset = ssPtr->offsetStart;
-    ssPtr->lit = ssPtr->litStart;
-    ssPtr->litLength = ssPtr->litLengthStart;
-    ssPtr->matchLength = ssPtr->matchLengthStart;
-    ssPtr->dumps = ssPtr->dumpsStart;
-
+    printf("litSum=%d litLengthSum=%d matchLengthSum=%d offCodeSum=%d\n", ssPtr->litSum, ssPtr->litLengthSum, ssPtr->matchLengthSum, ssPtr->offCodeSum);
     ssPtr->matchLengthSum = (1<<MLbits);
     ssPtr->litLengthSum = (1<<LLbits);
     ssPtr->litSum = (1<<Litbits);
@@ -121,6 +116,17 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
         ssPtr->offCodeFreq[i] = 1;
 }
 
+static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
+{
+    ssPtr->offset = ssPtr->offsetStart;
+    ssPtr->lit = ssPtr->litStart;
+    ssPtr->litLength = ssPtr->litLengthStart;
+    ssPtr->matchLength = ssPtr->matchLengthStart;
+    ssPtr->dumps = ssPtr->dumpsStart;
+
+    ZSTD_resetFreqs(ssPtr);
+}
+
 
 /* *************************************
 *  Context memory management
@@ -240,6 +246,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<<LLbits);
     zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<<MLbits);
  //   zc->seqStore.XXX = zc->seqStore.offCodeFreq + (1<<Offbits)*sizeof(U32);
+    ZSTD_resetFreqs(&zc->seqStore);
 
     zc->hbSize = 0;
     zc->stage = 0;

diff --git a/lib/zstd_opt.c b/lib/zstd_opt.c
index 72263b37d63ff6b124fd7cf1a63092a03cb3c53e..ef40648b4a87eacd3cc7c533d1e82f82a16f402b 100644 (file)
--- a/lib/zstd_opt.c
+++ b/lib/zstd_opt.c
@@ -33,7 +33,23 @@ typedef struct
 #define ZSTD_OPT_NUM   (1<<12)
 
 
-#define ZSTD_LIT_COST(len) 0 //(((len)<<3)+0)
+#define ZSTD_LIT_COST(len) 0 //(((len)<<3))
+
+const int tab32[32] = {
+     0,  9,  1, 10, 13, 21,  2, 29,
+    11, 14, 16, 18, 22, 25,  3, 30,
+     8, 12, 20, 28, 15, 17, 24,  7,
+    19, 27, 23,  6, 26,  5,  4, 31};
+
+int log2_32 (uint32_t value)
+{
+    value |= value >> 1;
+    value |= value >> 2;
+    value |= value >> 4;
+    value |= value >> 8;
+    value |= value >> 16;
+    return tab32[(uint32_t)(value*0x07C4ACDD) >> 27];
+}
 
 
 FORCE_INLINE size_t ZSTD_getLiteralPriceReal(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals)
@@ -79,14 +95,14 @@ FORCE_INLINE size_t ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, size_t litLeng
 #if 1
     return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals);
 #else
-    size_t lit_cost = 1 + (litLength<<3)+0;
+    size_t lit_cost = 1 + (litLength<<3);
     return lit_cost;
 #endif
 }
 
 
 
-FORCE_INLINE size_t ZSTD_getPriceReal(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
+FORCE_INLINE size_t ZSTD_getMatchPriceReal(seqStore_t* seqStorePtr, size_t offset, size_t matchLength)
 {
     size_t freq;
     size_t price = 0;
@@ -96,9 +112,12 @@ FORCE_INLINE size_t ZSTD_getPriceReal(seqStore_t* seqStorePtr, size_t litLength,
     /* match offset */
     BYTE offCode = (BYTE)ZSTD_highbit(offset) + 1;
     if (offset==0) 
-        offCodeBits = 0.0f;
-    else
-        offCodeBits = -log2((double)seqStorePtr->offCodeFreq[offCode]/(double)seqStorePtr->offCodeSum);
+        offCode = 0;
+ //   offCodeBits = -log2((double)seqStorePtr->offCodeFreq[offCode]/(double)seqStorePtr->offCodeSum);
+    offCodeBits = log2_32(seqStorePtr->offCodeSum) - log2_32(seqStorePtr->offCodeFreq[offCode]);
+ //   printf("offCodeBits=%.02f matchBits=%.02f dumpsPrice=%d sum=%d\n", offCodeBits, matchBits, (int)price, (int)freq);
+
+    offCodeBits += offCode;
 
     /* match Length */
     if (matchLength >= MaxML) {
@@ -110,24 +129,30 @@ FORCE_INLINE size_t ZSTD_getPriceReal(seqStore_t* seqStorePtr, size_t litLength,
             if (matchLength < (1<<15)) price += 16; else price += 24;
     }   }
     else freq = seqStorePtr->matchLengthFreq[matchLength];
-    matchBits = -log2((double)freq/(double)seqStorePtr->matchLengthSum);
+//    matchBits = -log2((double)freq/(double)seqStorePtr->matchLengthSum);
+    matchBits = log2_32(seqStorePtr->matchLengthSum) - log2_32(freq);
 
     freq = round(1.0f*(offCodeBits + matchBits + price));
-//        printf("litLength=%d litBits=%.02f litLenBits=%.02f offCodeBits=%.02f matchBits=%.02f dumpsPrice=%d sum=%d\n", (int)litLength, litBits, litLenBits, offCodeBits, matchBits, (int)price, (int)freq);
-    if (freq <= 0) return 1;
-
+//        printf("offCodeBits=%.02f matchBits=%.02f dumpsPrice=%d sum=%d\n", offCodeBits, matchBits, (int)price, (int)freq);
     return freq;
 }
 
+// zstd v0.5 beta level 23       1.94 MB/s    403 MB/s     40845550  38.95
+// zstd v0.5 beta level 24       1.80 MB/s    458 MB/s     40370570  38.50
+// zstd v0.5 beta level 23       1.10 MB/s       ERROR     40584556  38.70
+// zstd v0.5 beta level 24       0.87 MB/s       ERROR     40103205  38.25
+
 FORCE_INLINE size_t ZSTD_getPrice(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
 {
 #if 1
     size_t lit_cost = ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals);
-    size_t match_cost = /*MLbits +*/ ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
+    size_t match_cost2 = ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
+    size_t match_cost = ZSTD_getMatchPriceReal(seqStorePtr, offset, matchLength);
+  //  printf("old=%d new=%d\n", (int)match_cost2, (int)match_cost);
     return lit_cost + match_cost;
 #else
-    size_t lit_cost = (litLength<<3)+0;
-    size_t match_cost = /*MLbits +*/ ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
+    size_t lit_cost = (litLength<<3);
+    size_t match_cost = ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
     return lit_cost + match_cost;
 #endif
 }
@@ -979,7 +1004,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
 
             ZSTD_LOG_ENCODE("%d/%d: BEFORE_ENCODE literals=%d mlen=%d off=%d rep1=%d rep2=%d cur_rep=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2, cur_rep);
 
-#if 1
+#if 0
             if (rep_1 != cur_rep)
             {
                 printf("%d: ERROR rep_1=%d rep_2=%d cur_rep=%d\n", (int)(ip - base), (int)rep_1, (int)rep_2, cur_rep);