From: Yann Collet <cyan@fb.com>
Date: Thu, 10 May 2018 23:32:36 +0000 (-0700)
Subject: btopt : minor adjustment of update frequencies
X-Git-Tag: v1.3.5~3^2~55^2~5
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=74b1c75d64cb531526fd7c60f897f9bc0323986c;p=thirdparty%2Fzstd.git

btopt : minor adjustment of update frequencies
---

diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index eeb7b230a..25d3137a2 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -76,7 +76,7 @@ typedef struct {
     U32 rep[ZSTD_REP_NUM];
 } ZSTD_optimal_t;
 
-typedef enum { zop_none=0, zop_predef, zop_static } ZSTD_OptPrice_e;
+typedef enum { zop_dynamic=0, zop_predef, zop_static } ZSTD_OptPrice_e;
 
 typedef struct {
     /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 486c86931..7e7a6935f 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -12,7 +12,7 @@
 #include "zstd_opt.h"
 
 
-#define ZSTD_LITFREQ_ADD    2   /* scaling factor for litFreq, so that frequencies adapt faster to new stats. Also used for matchSum (?) */
+#define ZSTD_LITFREQ_ADD    2   /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
 #define ZSTD_FREQ_DIV       4   /* log factor when using previous stats to init next stats */
 #define ZSTD_MAX_PRICE     (1<<30)
 
@@ -32,24 +32,31 @@ static void ZSTD_setLog2Prices(optState_t* optPtr)
 static void ZSTD_rescaleFreqs(optState_t* const optPtr,
                               const BYTE* const src, size_t const srcSize)
 {
-    optPtr->priceType = zop_none;
+    optPtr->priceType = zop_dynamic;
 
-    if (optPtr->litLengthSum == 0) {  /* first init */
+    if (optPtr->litLengthSum == 0) {  /* first block : init */
         unsigned u;
-        if (srcSize <= 1024) optPtr->priceType = zop_predef;
+        if (srcSize <= 1024)   /* heuristic */
+            optPtr->priceType = zop_predef;
+
         assert(optPtr->symbolCosts != NULL);
         if (optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
-            optPtr->priceType = zop_static;
+            if (srcSize <= 8192)   /* heuristic */
+                optPtr->priceType = zop_static;
+            else {
+                assert(optPtr->priceType == zop_dynamic);
+            }
+
+
         }
 
-        assert(optPtr->litFreq!=NULL);
-        for (u=0; u<=MaxLit; u++)
-            optPtr->litFreq[u] = 0;
-        for (u=0; u<srcSize; u++)
-            optPtr->litFreq[src[u]]++;
+        assert(optPtr->litFreq != NULL);
+        {   unsigned max = MaxLit;
+            FSE_count(optPtr->litFreq, &max, src, srcSize);   /* use raw first block to init statistics */
+        }
         optPtr->litSum = 0;
         for (u=0; u<=MaxLit; u++) {
-            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u] >> ZSTD_FREQ_DIV);
+            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u] >> (ZSTD_FREQ_DIV+1));
             optPtr->litSum += optPtr->litFreq[u];
         }
 
@@ -63,7 +70,7 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
             optPtr->offCodeFreq[u] = 1;
         optPtr->offCodeSum = (MaxOff+1);
 
-    } else {
+    } else {   /* new block : re-use previous statistics, scaled down */
         unsigned u;
 
         optPtr->litSum = 0;
@@ -73,17 +80,17 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
         }
         optPtr->litLengthSum = 0;
         for (u=0; u<=MaxLL; u++) {
-            optPtr->litLengthFreq[u] = 1 + (optPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1));
+            optPtr->litLengthFreq[u] = 1 + (optPtr->litLengthFreq[u] >> ZSTD_FREQ_DIV);
             optPtr->litLengthSum += optPtr->litLengthFreq[u];
         }
         optPtr->matchLengthSum = 0;
         for (u=0; u<=MaxML; u++) {
-            optPtr->matchLengthFreq[u] = 1 + (optPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV);
+            optPtr->matchLengthFreq[u] = 1 + (optPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV);
             optPtr->matchLengthSum += optPtr->matchLengthFreq[u];
         }
         optPtr->offCodeSum = 0;
         for (u=0; u<=MaxOff; u++) {
-            optPtr->offCodeFreq[u] = 1 + (optPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV);
+            optPtr->offCodeFreq[u] = 1 + (optPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV);
             optPtr->offCodeSum += optPtr->offCodeFreq[u];
         }
     }