priceTable moved to heap

diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c
index c9847dafa4d64542819094d8a7af681cc00a8cbd..2eb1e6d2c4636b611d3f86321ff26c383ae3ffcb 100644 (file)
--- a/lib/zstd_compress.c
+++ b/lib/zstd_compress.c
@@ -174,7 +174,9 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     /* reserve table memory */
     const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
     const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog) + (1 << params.hashLog3)) * sizeof(U32);
-    const size_t neededSpace = tableSpace + (256*sizeof(U32)) + (3*blockSize) + ((1<<MLbits) + (1<<LLbits) + (1<<Offbits) + (1<<Litbits))*sizeof(U32);
+    const size_t neededSpace = tableSpace + (256*sizeof(U32)) + (3*blockSize)
+        + ((params.strategy == ZSTD_btopt) ? ((1<<MLbits) + (1<<LLbits) + (1<<Offbits) + (1<<Litbits))*sizeof(U32) + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)) : 0);
+
     if (zc->workSpaceSize < neededSpace) {
         free(zc->workSpace);
         zc->workSpace = malloc(neededSpace);
@@ -199,20 +201,21 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     zc->params = params;
     zc->blockSize = blockSize;
 
-    zc->seqStore.litFreq = (U32*) (zc->seqStore.buffer);
-    zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
-    zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<<LLbits);
-    zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<<MLbits);
-
-    zc->seqStore.offsetStart = zc->seqStore.offCodeFreq + (1<<Offbits);
+    zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
     zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (blockSize>>2));
     zc->seqStore.litStart = zc->seqStore.offCodeStart + (blockSize>>2);
     zc->seqStore.litLengthStart =  zc->seqStore.litStart + blockSize;
     zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2);
-    zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2);
-    // zc->seqStore.XXX = zc->seqStore.dumpsStart + (blockSize>>4);
-    zc->seqStore.litLengthSum = 0;
+    zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2); 
 
+    zc->seqStore.litFreq = (U32*)(zc->seqStore.dumpsStart + (blockSize>>2)); 
+    zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
+    zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<<LLbits);
+    zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<<MLbits);
+    zc->seqStore.matchTable = (ZSTD_match_t*)(zc->seqStore.offCodeFreq + (1<<Offbits));
+    zc->seqStore.priceTable = (ZSTD_optimal_t*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1);
+
+    zc->seqStore.litLengthSum = 0;
     zc->hbSize = 0;
     zc->stage = 0;
     zc->loadedDictEnd = 0;

diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h
index 68b66621f0e22d83328968e0ed99152eb7287e5e..4948e2397e0e0ed39677c1dc8fde471131f7de1a 100644 (file)
--- a/lib/zstd_internal.h
+++ b/lib/zstd_internal.h
@@ -64,6 +64,7 @@
     #define ZSTD_LOG_BLOCK(...)
 #endif
 
+#define ZSTD_OPT_NUM    (1<<12)
 #define ZSTD_DICT_MAGIC  0xEC30A435
 
 #define KB *(1 <<10)
@@ -165,6 +166,20 @@ MEM_STATIC unsigned ZSTD_highbit(U32 val)
 /*-*******************************************
 *  Private interfaces
 *********************************************/
+typedef struct {
+    U32 off;
+    U32 len;
+} ZSTD_match_t;
+
+typedef struct {
+    U32 price;
+    U32 off;
+    U32 mlen;
+    U32 litlen;
+    U32 rep;
+    U32 rep2;
+} ZSTD_optimal_t;
+
 typedef struct {
     void* buffer;
     U32*  offsetStart;
@@ -180,6 +195,8 @@ typedef struct {
     BYTE* dumpsStart;
     BYTE* dumps;
     /* opt */
+    ZSTD_optimal_t* priceTable;
+    ZSTD_match_t* matchTable;
     U32* matchLengthFreq;
     U32* litLengthFreq;
     U32* litFreq;

diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h
index a95f97b896211b3fcd7fe69632dc1abc9e628cdb..f61485987d1c055e603d71743b2c5fe9e2adfb86 100644 (file)
--- a/lib/zstd_opt.h
+++ b/lib/zstd_opt.h
@@ -269,8 +269,8 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx,
     const U32 mls = ctx->params.searchLength;
     const U32 sufficient_len = ctx->params.targetLength;
 
-    ZSTD_optimal_t opt[ZSTD_OPT_NUM+1];
-    ZSTD_match_t matches[ZSTD_OPT_NUM+1];
+    ZSTD_optimal_t* opt = seqStorePtr->priceTable;
+    ZSTD_match_t* matches = seqStorePtr->matchTable;
     const BYTE* inr;
     U32 cur, match_num, last_pos, litlen, price;
 
@@ -280,32 +280,6 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx,
     ZSTD_rescaleFreqs(seqStorePtr);
     if ((ip-prefixStart) < REPCODE_STARTVALUE) ip = prefixStart + REPCODE_STARTVALUE;
 
-#if ZSTD_OPT_DEBUG >= 3
-    size_t mostFrequent;
-    unsigned count[256], maxSymbolValue, usedSymbols = 0;
-    maxSymbolValue = 255;
-    mostFrequent = FSE_count(count, &maxSymbolValue, src, srcSize);
-    for (unsigned i=0; i<=maxSymbolValue; i++)
-        if (count[i]) usedSymbols++;
-
-    seqStorePtr->factor = ((usedSymbols <= 18) && (mostFrequent < (1<<14))) ? mostFrequent>>10 : 0; // helps RTF files
-    seqStorePtr->factor2 = (usedSymbols==256) && (mostFrequent > (1<<14));
-#endif
-
-#if 0
-    if (seqStorePtr->factor2)
-        printf("FACTOR2 usedSymbols==256;mostFrequent>(1<<14) maxSymbolValue=%d mostFrequent=%d usedSymbols=%d\n", maxSymbolValue, (int)mostFrequent, usedSymbols);
-    if (seqStorePtr->factor) {
-        printf("FACTOR1 usedSymbols<56;mostFrequent<(1<<14) maxSymbolValue=%d mostFrequent=%d usedSymbols=%d\n", maxSymbolValue, (int)mostFrequent, usedSymbols);
-#if 0
-        for (int i=0; i<256; i++)
-            if (count[i]) printf("%d=%d ", i, count[i]);
-        printf("\n");
-
-#endif
-    }
-#endif
-
     ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len);
 
     /* Match Loop */
@@ -619,8 +593,8 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx,
     const U32 mls = ctx->params.searchLength;
     const U32 sufficient_len = ctx->params.targetLength;
 
-    ZSTD_optimal_t opt[ZSTD_OPT_NUM+1];
-    ZSTD_match_t matches[ZSTD_OPT_NUM+1];
+    ZSTD_optimal_t* opt = seqStorePtr->priceTable;
+    ZSTD_match_t* matches = seqStorePtr->matchTable;
     const BYTE* inr;
     U32 cur, match_num, last_pos, litlen, price;
 
@@ -913,7 +887,31 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
             anchor = ip = ip + mlen;
         }
 
+#if 0
         /* check immediate repcode */
+        while ((anchor >= base + lowLimit + rep_2) && (anchor <= ilimit)) {
+            if ((anchor - rep_2) >= prefixStart) {
+                if (MEM_readMINMATCH(anchor) == MEM_readMINMATCH(anchor - rep_2))
+                    mlen = (U32)ZSTD_count(anchor+MINMATCHOPT, anchor - rep_2 + MINMATCHOPT, iend) + MINMATCHOPT;
+                else
+                    break;
+            } else {
+                const BYTE* repMatch = dictBase + ((anchor-base) - rep_2);
+                if ((repMatch + MINMATCHOPT <= dictEnd) && (MEM_readMINMATCH(anchor) == MEM_readMINMATCH(repMatch))) 
+                    mlen = (U32)ZSTD_count_2segments(anchor+MINMATCHOPT, repMatch+MINMATCHOPT, iend, dictEnd, prefixStart) + MINMATCHOPT;
+                else
+                    break;
+            }
+                   
+            offset = rep_2; rep_2 = rep_1; rep_1 = offset;   /* swap offset history */
+            ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2);
+            ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-MINMATCHOPT);
+            ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-MINMATCHOPT);
+            anchor += mlen;
+        }
+#else
+        /* check immediate repcode */
+        /* minimal correctness condition = while ((anchor >= prefixStart + REPCODE_STARTVALUE) && (anchor <= ilimit)) { */
         while ((anchor >= base + lowLimit + rep_2) && (anchor <= ilimit)) {
             const U32 repIndex = (U32)((anchor-base) - rep_2);
             const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
@@ -932,6 +930,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
             }
             break;
         }
+#endif
         if (anchor > ip) ip = anchor;
     }
 

diff --git a/lib/zstd_opt_internal.h b/lib/zstd_opt_internal.h
index f0a0e69e95256b8d64e4df408fd74091b0c4b73a..b980eab085249514ae77c73ae3bd45f84a02fb6e 100644 (file)
--- a/lib/zstd_opt_internal.h
+++ b/lib/zstd_opt_internal.h
@@ -40,25 +40,9 @@
 *  The optimal parser
 *********************************************/
 /*-  Constants  -*/
-#define ZSTD_OPT_NUM    (1<<12)
 #define ZSTD_FREQ_DIV   5
 
 
-typedef struct {
-    U32 off;
-    U32 len;
-} ZSTD_match_t;
-
-typedef struct {
-    U32 price;
-    U32 off;
-    U32 mlen;
-    U32 litlen;
-    U32 rep;
-    U32 rep2;
-} ZSTD_optimal_t;
-
-
 MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
 {
     unsigned u;