combined price function

diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h
index c3874e4eb33c8ae4e8d36b3119e84bbdc46f765e..d5266c4487846f58658f7818b6029979ad37643c 100644 (file)
--- a/lib/zstd_internal.h
+++ b/lib/zstd_internal.h
@@ -184,6 +184,8 @@ typedef struct {
     U32  realLitSum;
     U32  realSeqSum;
     U32  realRepSum;
+    U32  factor;
+    U32  factor2;
     U32  priceFunc;
 } seqStore_t;
 

diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h
index cfd17e5ac50f39cce5d3f1d6a1035f611cf9dd7c..5bb40e607fd37035443085f26102b79df9b77e58 100644 (file)
--- a/lib/zstd_opt.h
+++ b/lib/zstd_opt.h
@@ -48,12 +48,14 @@ FORCE_INLINE U32 ZSTD_GETPRICE(seqStore_t* seqStorePtr, U32 litLength, const BYT
 
     switch (seqStorePtr->priceFunc)
     {
-        default:
+        default:   
         case 0:
-            return price + ((seqStorePtr->litSum>>4) / seqStorePtr->litLengthSum);
+            return price + seqStorePtr->factor + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum));
         case 1:
-            return price + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum));
+            return price + seqStorePtr->factor + ((seqStorePtr->factor2) ? ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)) : 0);
         case 2:
+            return price + seqStorePtr->factor + ((seqStorePtr->factor2) ? ((seqStorePtr->litSum>>4) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)) : 0);
+        case 3:
             return price;
     }
 }
@@ -264,6 +266,33 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx,
     ZSTD_rescaleFreqs(seqStorePtr);
     if ((ip-base) < REPCODE_STARTVALUE) ip = base + REPCODE_STARTVALUE;
 
+
+    size_t mostFrequent;
+    unsigned count[256], maxSymbolValue, usedSymbols = 0;
+    maxSymbolValue = 255;
+    mostFrequent = FSE_count(count, &maxSymbolValue, src, srcSize);
+    for (unsigned i=0; i<=maxSymbolValue; i++)
+        if (count[i]) usedSymbols++;
+
+    seqStorePtr->factor = ((usedSymbols <= 18) && (mostFrequent < (1<<14))) ? mostFrequent>>10 : 0; // helps RTF files
+    seqStorePtr->factor2 = (usedSymbols==256) && (mostFrequent > (1<<14));
+
+#if 0
+    if (seqStorePtr->factor2)
+        printf("FACTOR2 usedSymbols==256;mostFrequent>(1<<14) maxSymbolValue=%d mostFrequent=%d usedSymbols=%d\n", maxSymbolValue, (int)mostFrequent, usedSymbols);
+    if (seqStorePtr->factor) {
+        printf("FACTOR1 usedSymbols<56;mostFrequent<(1<<14) maxSymbolValue=%d mostFrequent=%d usedSymbols=%d\n", maxSymbolValue, (int)mostFrequent, usedSymbols);
+#if 0
+        for (int i=0; i<256; i++)
+            if (count[i]) printf("%d=%d ", i, count[i]);
+        printf("\n");
+
+#endif
+    }
+#endif
+
+
+
     /* Match Loop */
     while (ip < ilimit) {
         U32 u;

diff --git a/lib/zstd_opt_internal.h b/lib/zstd_opt_internal.h
index c23d5d7401dc340b0d04d6132273a0f0181231ce..8628875299bb957a862835298b9d4783ad5cd4ad 100644 (file)
--- a/lib/zstd_opt_internal.h
+++ b/lib/zstd_opt_internal.h
@@ -43,7 +43,7 @@
 #define ZSTD_OPT_NUM    (1<<12)
 #define ZSTD_FREQ_START 1
 #define ZSTD_FREQ_STEP  1
-#define ZSTD_FREQ_DIV   4
+#define ZSTD_FREQ_DIV   5
 
 /*-  Debug  -*/
 #if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9
@@ -77,20 +77,20 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
     unsigned u;
 
     if (ssPtr->litLengthSum == 0) {
-        ssPtr->matchLengthSum = (1<<MLbits);
-        ssPtr->litLengthSum = (1<<LLbits);
-        ssPtr->litSum = (1<<Litbits);
-        ssPtr->offCodeSum = (1<<Offbits);
-        ssPtr->matchSum = 0;
+        ssPtr->matchLengthSum = ZSTD_FREQ_START*(1<<MLbits);
+        ssPtr->litLengthSum = ZSTD_FREQ_START*(1<<LLbits);
+        ssPtr->litSum = ZSTD_FREQ_START*(1<<Litbits);
+        ssPtr->offCodeSum = ZSTD_FREQ_START*(1<<Offbits);
+        ssPtr->matchSum = ZSTD_FREQ_START*ssPtr->litSum;
         
         for (u=0; u<=MaxLit; u++)
-            ssPtr->litFreq[u] = 1;
+            ssPtr->litFreq[u] = ZSTD_FREQ_START;
         for (u=0; u<=MaxLL; u++)
-            ssPtr->litLengthFreq[u] = 1;
+            ssPtr->litLengthFreq[u] = ZSTD_FREQ_START;
         for (u=0; u<=MaxML; u++)
-            ssPtr->matchLengthFreq[u] = 1;
+            ssPtr->matchLengthFreq[u] = ZSTD_FREQ_START;
         for (u=0; u<=MaxOff; u++)
-            ssPtr->offCodeFreq[u] = 1;
+            ssPtr->offCodeFreq[u] = ZSTD_FREQ_START;
     } else {
         ssPtr->matchLengthSum = 0;
         ssPtr->litLengthSum = 0;