Update function names for consistency

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1490b044a216e89baf295910290bc102ec56f265..1a430e544b08f538978dba8d014ca6a4f775acbe 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2192,6 +2192,32 @@ static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams)
     return (cctxParams->targetCBlockSize != 0);
 }
 
+/* Pseudocode algorithm for finding the optimal partition:
+ * Given n sequences:
+ * Let epsilon = 1
+ * 
+
+typedef struct {
+    size_t startIdx;
+    size_t endIdx;
+} ZSTD_sequenceWindow;
+
+size_t ZSTD_sequenceWindow_moveStartIdx(ZSTD_sequenceWindow* sequenceWindow) {
+    ++sequenceWindow->startIdx;
+}
+
+size_t ZSTD_sequenceWindow_moveEndIdx(ZSTD_sequenceWindow* sequenceWindow) {
+    ++sequenceWindow->endIdx;
+}
+
+size_t ZSTD_sequenceWindow_currentCost(ZSTD_sequenceWindow* sequenceWindow) {
+    return 0;
+}
+
+/* ZSTD_buildSequencesStatistics():
+ * Returns the size of the statistics for a given set of sequences, or a ZSTD error code
+ */
+
 MEM_STATIC size_t
 ZSTD_buildSequencesStatistics(const BYTE* const ofCodeTable,
                               const BYTE* const llCodeTable, 
@@ -2325,7 +2351,6 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr,
     FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
     FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
     FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
-    U32 LLtype, Offtype, MLtype;   /* compressed, raw or rle */
     U32 entropyStatisticsSize;
     const seqDef* const sequences = seqStorePtr->sequencesStart;
     const size_t nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
@@ -2335,7 +2360,6 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr,
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + dstCapacity;
     BYTE* op = ostart;
-    BYTE* seqHead;
     BYTE* lastNCount = NULL;
 
     entropyWorkspace = count + (MaxSeq + 1);
@@ -2528,16 +2552,6 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
 
 typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
 
-static U32 countSeqStoreLiteralsBytes2(const seqStore_t* seqStore) {
-    U32 literalsBytes = 0;
-    U32 nbSeqs = seqStore->sequences - seqStore->sequencesStart;
-    for (int i = 0; i < nbSeqs; ++i) {
-        seqDef seq = seqStore->sequencesStart[i];
-        literalsBytes += seq.litLength;
-    }
-    return literalsBytes;
-}
-
 static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
 {
     ZSTD_matchState_t* const ms = &zc->blockState.matchState;
@@ -2761,19 +2775,19 @@ static void ZSTD_confirmRepcodesAndEntropyTables(ZSTD_CCtx* zc)
 
 /* Writes the block header */
 static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) {
-    DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock);
     U32 const cBlockHeader = cSize == 1 ?
                         lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
                         lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
     MEM_writeLE24(op, cBlockHeader);
+    DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock);
 }
 
-/** ZSTD_buildSuperBlockEntropy_literal() :
+/** ZSTD_buildBlockEntropyStats_literals() :
  *  Builds entropy for the super-block literals.
  *  Stores literals block type (raw, rle, compressed, repeat) and
  *  huffman description table to hufMetadata.
  *  @return : size of huffman description table or error code */
-static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize,
+static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
                                             const ZSTD_hufCTables_t* prevHuf,
                                                   ZSTD_hufCTables_t* nextHuf,
                                                   ZSTD_hufCTablesMetadata_t* hufMetadata,
@@ -2791,7 +2805,7 @@ static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSiz
     unsigned huffLog = HUF_TABLELOG_DEFAULT;
     HUF_repeat repeat = prevHuf->repeatMode;
 
-    DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize);
+    DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize);
 
     /* Prepare nextEntropy assuming reusing the existing table */
     ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
@@ -2871,11 +2885,11 @@ static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSiz
     }
 }
 
-/** ZSTD_buildSuperBlockEntropy_sequences() :
+/** ZSTD_buildBlockEntropyStats_sequences() :
  *  Builds entropy for the super-block sequences.
  *  Stores symbol compression modes and fse table to fseMetadata.
  *  @return : size of fse tables or error code */
-static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr,
+static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
                                               const ZSTD_fseCTables_t* prevEntropy,
                                                     ZSTD_fseCTables_t* nextEntropy,
                                               const ZSTD_CCtx_params* cctxParams,
@@ -2902,11 +2916,10 @@ static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr,
     BYTE* const ostart = fseMetadata->fseTablesBuffer;
     BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
     BYTE* op = ostart;
-    U32 entropyStatisticsSize;
     BYTE* lastNCount = NULL;
 
     assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE));
-    DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq);
+    DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq);
     ZSTD_memset(workspace, 0, wkspSize);
 
     fseMetadata->lastCountSize = 0;
@@ -2918,10 +2931,10 @@ static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr,
 }
 
 
-/** ZSTD_buildSuperBlockEntropy() :
+/** ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the super-block.
  *  @return : 0 on success or error code */
-size_t ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
+size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
                              const ZSTD_entropyCTables_t* prevEntropy,
                                    ZSTD_entropyCTables_t* nextEntropy,
                              const ZSTD_CCtx_params* cctxParams,
@@ -2929,21 +2942,21 @@ size_t ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
                                    void* workspace, size_t wkspSize)
 {
     size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
-    DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy");
+    DEBUGLOG(5, "ZSTD_buildBlockEntropyStats");
     entropyMetadata->hufMetadata.hufDesSize =
-        ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize,
+        ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
                                             &prevEntropy->huf, &nextEntropy->huf,
                                             &entropyMetadata->hufMetadata,
                                             ZSTD_disableLiteralsCompression(cctxParams),
                                             workspace, wkspSize);
-    FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed");
+    FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
     entropyMetadata->fseMetadata.fseTablesSize =
-        ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr,
+        ZSTD_buildBlockEntropyStats_sequences(seqStorePtr,
                                               &prevEntropy->fse, &nextEntropy->fse,
                                               cctxParams,
                                               &entropyMetadata->fseMetadata,
                                               workspace, wkspSize);
-    FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed");
+    FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed");
     return 0;
 }
 
@@ -3064,7 +3077,7 @@ size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
 static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(const ZSTD_CCtx* zc, seqStore_t* seqStore) {
     ZSTD_entropyCTablesMetadata_t entropyMetadata;
     size_t estimatedSize;
-    FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(seqStore,
+    FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
                     &zc->blockState.prevCBlock->entropy,
                     &zc->blockState.nextCBlock->entropy,
                     &zc->appliedParams,
@@ -3078,10 +3091,12 @@ static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(const ZSTD_CCtx
     return estimatedSize;
 }
 
-static U32 countSeqStoreLiteralsBytes(const seqStore_t* seqStore) {
-    U32 literalsBytes = 0;
-    U32 nbSeqs = seqStore->sequences - seqStore->sequencesStart;
-    for (int i = 0; i < nbSeqs; ++i) {
+/* Returns literals bytes represented in a seqStore */
+static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* seqStore) {
+    size_t literalsBytes = 0;
+    size_t nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t i;
+    for (i = 0; i < nbSeqs; ++i) {
         seqDef seq = seqStore->sequencesStart[i];
         literalsBytes += seq.litLength;
         if (i == seqStore->longLengthPos && seqStore->longLengthID == 1) {
@@ -3091,10 +3106,12 @@ static U32 countSeqStoreLiteralsBytes(const seqStore_t* seqStore) {
     return literalsBytes;
 }
 
-static U32 countSeqStoreMatchBytes(const seqStore_t* seqStore) {
-    U32 matchBytes = 0;
-    U32 nbSeqs = seqStore->sequences - seqStore->sequencesStart;
-    for (int i = 0; i < nbSeqs; ++i) {
+/* Returns match bytes represented in a seqStore */
+static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* seqStore) {
+    size_t matchBytes = 0;
+    size_t nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t i;
+    for (i = 0; i < nbSeqs; ++i) {
         seqDef seq = seqStore->sequencesStart[i];
         matchBytes += seq.matchLength + MINMATCH;
         if (i == seqStore->longLengthPos && seqStore->longLengthID == 2) {
@@ -3104,12 +3121,17 @@ static U32 countSeqStoreMatchBytes(const seqStore_t* seqStore) {
     return matchBytes;
 }
 
-static void splitSeqStores(const seqStore_t* originalSeqStore,
+/* ZSTD_splitSeqStores():
+ * Splits the original seqStore into two, with nbSeqFirstHalf sequences in the first
+ * seqStore, and the remainder in the second. 
+ */
+static void ZSTD_splitSeqStores(const seqStore_t* originalSeqStore,
                            seqStore_t* firstSeqStore, seqStore_t* secondSeqStore,
                            size_t nbSeqFirstHalf) {
     
     BYTE* const litEnd = originalSeqStore->lit;
     seqDef* const seqEnd = originalSeqStore->sequences;
+    U32 literalsBytesFirstHalf;
     *firstSeqStore = *originalSeqStore;
     *secondSeqStore = *originalSeqStore;
 
@@ -3124,7 +3146,7 @@ static void splitSeqStores(const seqStore_t* originalSeqStore,
 
     firstSeqStore->sequences = firstSeqStore->sequencesStart+nbSeqFirstHalf;
 
-    U32 literalsBytesFirstHalf = countSeqStoreLiteralsBytes(firstSeqStore);
+    literalsBytesFirstHalf = ZSTD_countSeqStoreLiteralsBytes(firstSeqStore);
     firstSeqStore->lit = firstSeqStore->litStart+literalsBytesFirstHalf;
 
     secondSeqStore->sequencesStart += nbSeqFirstHalf;
@@ -3134,15 +3156,21 @@ static void splitSeqStores(const seqStore_t* originalSeqStore,
     secondSeqStore->llCode += nbSeqFirstHalf;
     secondSeqStore->mlCode += nbSeqFirstHalf;
     secondSeqStore->ofCode += nbSeqFirstHalf;
-    DEBUGLOG(2, "Split into: %u and %u", (U32)(firstSeqStore->sequences - firstSeqStore->sequencesStart),
-                                         (U32)(secondSeqStore->sequences - secondSeqStore->sequencesStart));
+    DEBUGLOG(2, "Split into: %u and %u seqs", (U32)(firstSeqStore->sequences - firstSeqStore->sequencesStart),
+                                              (U32)(secondSeqStore->sequences - secondSeqStore->sequencesStart));
 }
 
-#define NB_SPLIT_POINTS_TO_TEST 2
-static int setUpSeqStores(ZSTD_CCtx* zc,
+/* ZSTD_deriveSplitSeqstores()
+ * Simple block splitting approach: test a set number of fixed block partitions. 
+ * For now, just a single split down the middle of the block.
+ * 
+ * Returns 1 if the a split was performed, 0 if not.
+ */
+#define NB_BLOCK_SEGMENTS_TO_TEST 2
+static int ZSTD_deriveSplitSeqstores(ZSTD_CCtx* zc,
                            seqStore_t* firstSeqStore, seqStore_t* secondSeqStore,
-                           U32 nbSeq, U32 srcSize) {
-    size_t increment = nbSeq/NB_SPLIT_POINTS_TO_TEST + 1;
+                           U32 nbSeq) {
+    size_t increment = nbSeq/NB_BLOCK_SEGMENTS_TO_TEST + 1;
     size_t estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(zc, &zc->seqStore);
     size_t minEstimatedCSize = estimatedOriginalSize;
     size_t minEstimatedCSizeIdx = 0;
@@ -3152,6 +3180,7 @@ static int setUpSeqStores(ZSTD_CCtx* zc,
         return 0;
     }
     
+    DEBUGLOG(2, "Estimated original block size is: %zu", estimatedOriginalSize); 
     DEBUGLOG(2, "total nbseq: %u, increment: %zu", nbSeq, increment);
     for (i = increment; i < nbSeq; i += increment) {
         /* Check that splitting would actually improve compression. Return 0 if not */
@@ -3159,12 +3188,11 @@ static int setUpSeqStores(ZSTD_CCtx* zc,
         size_t estimatedSecondHalfSize;
         size_t estimatedSplitBlocksCompressedSize;
         size_t nbSeqFirstHalf = i;
-        splitSeqStores(&zc->seqStore, firstSeqStore, secondSeqStore, nbSeqFirstHalf);
+        ZSTD_splitSeqStores(&zc->seqStore, firstSeqStore, secondSeqStore, nbSeqFirstHalf);
         estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(zc, firstSeqStore);
         estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(zc, secondSeqStore);
         estimatedSplitBlocksCompressedSize = estimatedFirstHalfSize + estimatedSecondHalfSize;    
-
-        DEBUGLOG(2, "Estimated original block size is: %zu", estimatedOriginalSize);              
+             
         DEBUGLOG(2, "Estimated split block size is: %zu - split: %zu - %zu", estimatedSplitBlocksCompressedSize, estimatedFirstHalfSize, estimatedSecondHalfSize);
         if (estimatedSplitBlocksCompressedSize < minEstimatedCSize) {
             minEstimatedCSizeIdx = i;
@@ -3174,7 +3202,7 @@ static int setUpSeqStores(ZSTD_CCtx* zc,
 
     if (minEstimatedCSizeIdx != 0) {
         DEBUGLOG(2, "WILL SPLIT");
-        splitSeqStores(&zc->seqStore, firstSeqStore, secondSeqStore, minEstimatedCSizeIdx);
+        ZSTD_splitSeqStores(&zc->seqStore, firstSeqStore, secondSeqStore, minEstimatedCSizeIdx);
         return 1;
     } else {
         DEBUGLOG(2, "NOT SPLITTING");
@@ -3182,6 +3210,13 @@ static int setUpSeqStores(ZSTD_CCtx* zc,
     }
 }
 
+/* ZSTD_compressSequences_singleBlock():
+ * Compresses a seqStore into a block with a block header, into the buffer dst.
+ * 
+ * Returns the size of that block or a ZSTD error code
+ */
+
+/* TODO: Migrate compressBlock_internal and compressSequences_internal to use this as well */
 static size_t ZSTD_compressSequences_singleBlock(ZSTD_CCtx* zc, seqStore_t* seqStore,
                                              void* dst, size_t dstCapacity,
                                              const void* src, size_t srcSize,
@@ -3226,7 +3261,6 @@ static size_t ZSTD_compressSequences_singleBlock(ZSTD_CCtx* zc, seqStore_t* seqS
         FORWARD_IF_ERROR(cSize, "RLE compress block failed");
         DEBUGLOG(2, "1: Writing out RLE block, size: %zu", cSize);
     } else {
-        U32 cBlockHeader;
         /* Error checking and repcodes update */
         ZSTD_confirmRepcodesAndEntropyTables(zc);
         writeBlockHeader(op, cSeqsSize, srcSize, lastBlock);
@@ -3236,52 +3270,56 @@ static size_t ZSTD_compressSequences_singleBlock(ZSTD_CCtx* zc, seqStore_t* seqS
     return cSize;
 }
 
+/* ZSTD_compressBlock_splitBlock():
+ * Attempts to split a given block into multiple (currently 2) blocks to improve compression ratio.
+ * 
+ * Returns 0 if it would not be advantageous to split the block. Otherwise, returns the combined size
+ * of the multiple blocks, or a ZSTD error code.
+ */
 static size_t ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
                                         void* dst, size_t dstCapacity,
-                                        const void* src, size_t srcSize, U32 frame, U32 lastBlock, U32 nbSeq) {
-    const U32 rleMaxLength = 25;
+                                        const void* src, size_t srcSize, U32 lastBlock, U32 nbSeq) {
     size_t cSize;
     const BYTE* ip = (const BYTE*)src;
     BYTE* op = (BYTE*)dst;
+    seqStore_t firstHalfSeqStore;
+    seqStore_t secondHalfSeqStore;
+    size_t cSizeFirstHalf;
+    size_t cSizeSecondHalf;
     DEBUGLOG(5, "ZSTD_compressBlock_splitBlock (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
                 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
                 (unsigned)zc->blockState.matchState.nextToUpdate);
-
-    /* Attempt block splitting here */
     DEBUGLOG(3, "Block size pre-split is: %zu - lastBlock: %u", srcSize, lastBlock);
-    DEBUGLOG(3, "srcSize: %zu seq store size: %u", srcSize, countSeqStoreLiteralsBytes(&zc->seqStore) + countSeqStoreMatchBytes(&zc->seqStore));
-    seqStore_t firstHalfSeqStore;
-    seqStore_t secondHalfSeqStore;
-    if (setUpSeqStores(zc, &firstHalfSeqStore, &secondHalfSeqStore, nbSeq, srcSize) != 1) {
+    DEBUGLOG(3, "srcSize: %zu seq store size: %zu", srcSize, ZSTD_countSeqStoreLiteralsBytes(&zc->seqStore) + ZSTD_countSeqStoreMatchBytes(&zc->seqStore));
+    /* Attempt block splitting here */
+    if (!ZSTD_deriveSplitSeqstores(zc, &firstHalfSeqStore, &secondHalfSeqStore, nbSeq)) {
+        /* Not advantageous to split blocks */
         return 0;
     }
 
     assert((U32)(firstHalfSeqStore.lit - firstHalfSeqStore.litStart) + (U32)(secondHalfSeqStore.lit - secondHalfSeqStore.litStart) == (U32)(zc->seqStore.lit - zc->seqStore.litStart));
     assert((U32)(firstHalfSeqStore.sequences - firstHalfSeqStore.sequencesStart) + (U32)(secondHalfSeqStore.sequences - secondHalfSeqStore.sequencesStart)
-            == (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart));
+        == (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart));
 
-    size_t cSizeFirstHalf;
-    size_t cSizeSecondHalf;
-
-    size_t literalsBytesFirstHalf = countSeqStoreLiteralsBytes(&firstHalfSeqStore);
-    size_t srcBytesFirstHalf = literalsBytesFirstHalf + countSeqStoreMatchBytes(&firstHalfSeqStore);
-    size_t srcBytesSecondHalf = srcSize - srcBytesFirstHalf;
-    DEBUGLOG(3, "literals bytes first half: %zu literals bytes second half: %u, orig: %u", literalsBytesFirstHalf, countSeqStoreLiteralsBytes(&secondHalfSeqStore), countSeqStoreLiteralsBytes(&zc->seqStore));
-    DEBUGLOG(3, "match bytes first half: %u match bytes second half: %u, orig: %u", countSeqStoreMatchBytes(&firstHalfSeqStore), countSeqStoreMatchBytes(&secondHalfSeqStore), countSeqStoreMatchBytes(&zc->seqStore));
-    DEBUGLOG(2, "Src bytes first half: %zu src bytes second half: %zu", srcBytesFirstHalf, srcBytesSecondHalf);
-
-    cSizeFirstHalf = ZSTD_compressSequences_singleBlock(zc, &firstHalfSeqStore, op, dstCapacity, ip, srcBytesFirstHalf, 0 /* lastBlock */);
     {
-        int i;
-        for (i = 0; i < ZSTD_REP_NUM; ++i)
-            zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i];
-        ip += srcBytesFirstHalf;
-        op += cSizeFirstHalf;
-        dstCapacity -= cSizeFirstHalf;
-    }
-    cSizeSecondHalf = ZSTD_compressSequences_singleBlock(zc, &secondHalfSeqStore, op, dstCapacity, ip, srcBytesSecondHalf, lastBlock /* lastBlock */);
-    DEBUGLOG(2, "cSizeFirstHalf: %zu cSizeSecondHalf: %zu", cSizeFirstHalf, cSizeSecondHalf);
-    cSize = cSizeFirstHalf + cSizeSecondHalf;
+        size_t literalsBytesFirstHalf = ZSTD_countSeqStoreLiteralsBytes(&firstHalfSeqStore);
+        size_t srcBytesFirstHalf = literalsBytesFirstHalf + ZSTD_countSeqStoreMatchBytes(&firstHalfSeqStore);
+        size_t srcBytesSecondHalf = srcSize - srcBytesFirstHalf;
+        DEBUGLOG(3, "literals bytes first half: %zu literals bytes second half: %zu, orig: %zu", literalsBytesFirstHalf, ZSTD_countSeqStoreLiteralsBytes(&secondHalfSeqStore), ZSTD_countSeqStoreLiteralsBytes(&zc->seqStore));
+        DEBUGLOG(3, "match bytes first half: %zu match bytes second half: %zu, orig: %zu", ZSTD_countSeqStoreMatchBytes(&firstHalfSeqStore), ZSTD_countSeqStoreMatchBytes(&secondHalfSeqStore), ZSTD_countSeqStoreMatchBytes(&zc->seqStore));
+        DEBUGLOG(2, "Src bytes first half: %zu src bytes second half: %zu", srcBytesFirstHalf, srcBytesSecondHalf);
+
+        cSizeFirstHalf = ZSTD_compressSequences_singleBlock(zc, &firstHalfSeqStore, op, dstCapacity, ip, srcBytesFirstHalf, 0 /* lastBlock */);
+        {   /* Perform necessary updates before compressing next block */
+            ZSTD_memcpy(zc->blockState.nextCBlock->rep, zc->blockState.prevCBlock->rep, ZSTD_REP_NUM);
+            ip += srcBytesFirstHalf;
+            op += cSizeFirstHalf;
+            dstCapacity -= cSizeFirstHalf;
+        }
+        cSizeSecondHalf = ZSTD_compressSequences_singleBlock(zc, &secondHalfSeqStore, op, dstCapacity, ip, srcBytesSecondHalf, lastBlock /* lastBlock */);
+        DEBUGLOG(2, "cSizeFirstHalf: %zu cSizeSecondHalf: %zu", cSizeFirstHalf, cSizeSecondHalf);
+        cSize = cSizeFirstHalf + cSizeSecondHalf;
+    }
     return cSize;
 }
 
@@ -3311,7 +3349,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
     zc->appliedParams.splitBlocks = 1; /* remove */
     if (zc->appliedParams.splitBlocks && nbSeq >= 2) {
         size_t splitBlocksCompressedSize;
-        splitBlocksCompressedSize = ZSTD_compressBlock_splitBlock(zc, dst, dstCapacity, src, srcSize, frame, lastBlock, nbSeq);
+        splitBlocksCompressedSize = ZSTD_compressBlock_splitBlock(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq);
         if (splitBlocksCompressedSize != 0) {
             return splitBlocksCompressedSize;
         }

diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 48b6c084109ff04b3c52c417cb497c69dbc2778e..33c653e27e052da2db5527a2aec1688ec842c853 100644 (file)
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -89,7 +89,7 @@ typedef struct {
  *  Stores Literals Block Type for a super-block in hType, and
  *  huffman tree description in hufDesBuffer.
  *  hufDesSize refers to the size of huffman tree description in bytes.
- *  This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */
+ *  This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */
 typedef struct {
     symbolEncodingType_e hType;
     BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
@@ -100,7 +100,7 @@ typedef struct {
  *  Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
  *  fse tables in fseTablesBuffer.
  *  fseTablesSize refers to the size of fse tables in bytes.
- *  This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */
+ *  This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */
 typedef struct {
     symbolEncodingType_e llType;
     symbolEncodingType_e ofType;
@@ -115,10 +115,10 @@ typedef struct {
     ZSTD_fseCTablesMetadata_t fseMetadata;
 } ZSTD_entropyCTablesMetadata_t;
 
-/** ZSTD_buildSuperBlockEntropy() :
+/** ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the super-block.
  *  @return : 0 on success or error code */
-size_t ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
+size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
                              const ZSTD_entropyCTables_t* prevEntropy,
                                    ZSTD_entropyCTables_t* nextEntropy,
                              const ZSTD_CCtx_params* cctxParams,

diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c
index d04566da77830a978c2a1e3da647eac4ef7a535a..eb0abb0cacb02dff4251a6b5dbbdc851829c532e 100644 (file)
--- a/lib/compress/zstd_compress_superblock.c
+++ b/lib/compress/zstd_compress_superblock.c
@@ -727,7 +727,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
                                unsigned lastBlock) {
     ZSTD_entropyCTablesMetadata_t entropyMetadata;
 
-    FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore,
+    FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
           &zc->blockState.prevCBlock->entropy,
           &zc->blockState.nextCBlock->entropy,
           &zc->appliedParams,

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 9f753911d0da241fa4457ff30dc3bbeff1161031..15139501bea196fbe770ffdc1c824341ae0ef9ec 100644 (file)
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -763,7 +763,7 @@ size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSiz
 static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
                           const void* src, size_t srcSize)
 {
-    DEBUGLOG(2, "ZSTD_copyRawBlock: %u", srcSize);
+    DEBUGLOG(5, "ZSTD_copyRawBlock");
     RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
     if (dst == NULL) {
         if (srcSize == 0) return 0;
@@ -847,7 +847,6 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
 
     /* Loop on each block */
     while (1) {
-        DEBUGLOG(2, "Remaining dstCap: %u", (size_t)(oend-op));
         size_t decodedSize;
         blockProperties_t blockProperties;
         size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
@@ -876,10 +875,8 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         if (ZSTD_isError(decodedSize)) return decodedSize;
         if (dctx->validateChecksum)
             XXH64_update(&dctx->xxhState, op, decodedSize);
-        if (decodedSize != 0) {
-            DEBUGLOG(2, "Decoded: %u", decodedSize);
+        if (decodedSize != 0)
             op += decodedSize;
-        }
         assert(ip != NULL);
         ip += cBlockSize;
         remainingSrcSize -= cBlockSize;
@@ -1192,7 +1189,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
             }
             FORWARD_IF_ERROR(rSize, "");
             RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
-            DEBUGLOG(2, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
+            DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
             dctx->decodedSize += rSize;
             if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize);
             dctx->previousDstEnd = (char*)dst + rSize;

diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 56e4b5ef87ae8beffd7e507342f60f62575b9fcc..b71bc20d65b4e8e72dca056329cef42151a35088 100644 (file)
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -775,9 +775,6 @@ size_t ZSTD_execSequenceEnd(BYTE* op,
 
     /* bounds checks : careful of address space overflow in 32-bit mode */
     RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
-    DEBUGLOG(2, "sequence length: %u", sequenceLength);
-    DEBUGLOG(2, "oLitEnd: %u iLitEnd: %u match: %u", oLitEnd, iLitEnd, match);
-    DEBUGLOG(2, "seq ll: %u, condition: %u", sequence.litLength, (size_t)(litLimit - *litPtr));
     RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
     assert(op < op + sequenceLength);
     assert(oLitEnd < op + sequenceLength);
@@ -853,13 +850,8 @@ size_t ZSTD_execSequence(BYTE* op,
     op = oLitEnd;
     *litPtr = iLitEnd;   /* update for next sequence */
 
-    
     /* Copy Match */
     if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
-        /*DEBUGLOG(2, "oLitEnd: %u, oMatchEnd: %u iLitEnd: %u matchPos: %u", oLitEnd, oMatchEnd, iLitEnd, match);
-        DEBUGLOG(2, "off: %u ml: %u ll: %u", sequence.offset, sequence.matchLength, sequence.litLength);
-        DEBUGLOG(2, "first condition: %u", (size_t)(oLitEnd - prefixStart));
-        DEBUGLOG(2, "break condition: %u", (size_t)(oLitEnd - virtualStart));*/
         /* offset beyond prefix -> go into extDict */
         RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
         match = dictEnd + (match - prefixStart);
@@ -1218,9 +1210,6 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
 
     /* last literal segment */
     {   size_t const lastLLSize = litEnd - litPtr;
-        if (lastLLSize > (size_t)(oend-op)) {
-            DEBUGLOG(2, "too small lastll");
-        }
         RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
         if (op != NULL) {
             ZSTD_memcpy(op, litPtr, lastLLSize);
@@ -1469,7 +1458,7 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
      * (note: but it could be evaluated from current-lowLimit)
      */
     ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));
-    DEBUGLOG(2, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
+    DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
 
     RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");