btopt : minor adjustment of update frequencies

diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index eeb7b230a317d8e5791389557c297a26ac31e6bc..25d3137a2d3a0518a8173e2dbb650a184e058b49 100644 (file)
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -76,7 +76,7 @@ typedef struct {
     U32 rep[ZSTD_REP_NUM];
 } ZSTD_optimal_t;
 
-typedef enum { zop_none=0, zop_predef, zop_static } ZSTD_OptPrice_e;
+typedef enum { zop_dynamic=0, zop_predef, zop_static } ZSTD_OptPrice_e;
 
 typedef struct {
     /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 486c869310526d88bb846f37fd3f1267862c102f..7e7a6935f5519901d24189514d6da71eee5f8183 100644 (file)
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -12,7 +12,7 @@
 #include "zstd_opt.h"
 
 
-#define ZSTD_LITFREQ_ADD    2   /* scaling factor for litFreq, so that frequencies adapt faster to new stats. Also used for matchSum (?) */
+#define ZSTD_LITFREQ_ADD    2   /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
 #define ZSTD_FREQ_DIV       4   /* log factor when using previous stats to init next stats */
 #define ZSTD_MAX_PRICE     (1<<30)
 
@@ -32,24 +32,31 @@ static void ZSTD_setLog2Prices(optState_t* optPtr)
 static void ZSTD_rescaleFreqs(optState_t* const optPtr,
                               const BYTE* const src, size_t const srcSize)
 {
-    optPtr->priceType = zop_none;
+    optPtr->priceType = zop_dynamic;
 
-    if (optPtr->litLengthSum == 0) {  /* first init */
+    if (optPtr->litLengthSum == 0) {  /* first block : init */
         unsigned u;
-        if (srcSize <= 1024) optPtr->priceType = zop_predef;
+        if (srcSize <= 1024)   /* heuristic */
+            optPtr->priceType = zop_predef;
+
         assert(optPtr->symbolCosts != NULL);
         if (optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
-            optPtr->priceType = zop_static;
+            if (srcSize <= 8192)   /* heuristic */
+                optPtr->priceType = zop_static;
+            else {
+                assert(optPtr->priceType == zop_dynamic);
+            }
+
+
         }
 
-        assert(optPtr->litFreq!=NULL);
-        for (u=0; u<=MaxLit; u++)
-            optPtr->litFreq[u] = 0;
-        for (u=0; u<srcSize; u++)
-            optPtr->litFreq[src[u]]++;
+        assert(optPtr->litFreq != NULL);
+        {   unsigned max = MaxLit;
+            FSE_count(optPtr->litFreq, &max, src, srcSize);   /* use raw first block to init statistics */
+        }
         optPtr->litSum = 0;
         for (u=0; u<=MaxLit; u++) {
-            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u] >> ZSTD_FREQ_DIV);
+            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u] >> (ZSTD_FREQ_DIV+1));
             optPtr->litSum += optPtr->litFreq[u];
         }
 
@@ -63,7 +70,7 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
             optPtr->offCodeFreq[u] = 1;
         optPtr->offCodeSum = (MaxOff+1);
 
-    } else {
+    } else {   /* new block : re-use previous statistics, scaled down */
         unsigned u;
 
         optPtr->litSum = 0;
@@ -73,17 +80,17 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
         }
         optPtr->litLengthSum = 0;
         for (u=0; u<=MaxLL; u++) {
-            optPtr->litLengthFreq[u] = 1 + (optPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1));
+            optPtr->litLengthFreq[u] = 1 + (optPtr->litLengthFreq[u] >> ZSTD_FREQ_DIV);
             optPtr->litLengthSum += optPtr->litLengthFreq[u];
         }
         optPtr->matchLengthSum = 0;
         for (u=0; u<=MaxML; u++) {
-            optPtr->matchLengthFreq[u] = 1 + (optPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV);
+            optPtr->matchLengthFreq[u] = 1 + (optPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV);
             optPtr->matchLengthSum += optPtr->matchLengthFreq[u];
         }
         optPtr->offCodeSum = 0;
         for (u=0; u<=MaxOff; u++) {
-            optPtr->offCodeFreq[u] = 1 + (optPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV);
+            optPtr->offCodeFreq[u] = 1 + (optPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV);
             optPtr->offCodeSum += optPtr->offCodeFreq[u];
         }
     }