fixed Multi-threaded compression

MT compression generates a single frame.
Multi-threading operates by breaking the frames into independent sections.
But from a decoder perspective, there is no difference :
it's just a suite of blocks.

Problem is, decoder preserves repCodes from previous block to start decoding next block.
This is also valid between sections, since they are no different than changing block.

Previous version would incorrectly initialize repcodes to their default value at the beginning of each section.
When using them, there was a mismatch between encoder (default values) and decoder (values from previous block).

This change ensures that repcodes won't be used at the beginning of a new section.
It works by setting them to 0.
This only works with regular (single segment) variants : extDict variants will fail !
Fortunately, sections beyond the 1st one belong to this category.

To be checked : btopt strategy.
This change was only validated from fast to btlazy2 strategies.

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 96e0577587ddad676c7843911e12eb16de0a9ce8..4b56ce1a2273ba99883ef33a2f88b5f5d4adc6a1 100644 (file)
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -267,4 +267,13 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
 }
 
 
+/* hidden functions */
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ *        do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
+
+
 #endif   /* ZSTD_CCOMMON_H_MODULE */

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index d4800dce12dc59af3f1eaf46467d37556b381af1..84a4a0216ed25964b5f681d58e845534cc7404d0 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -317,6 +317,14 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     }
 }
 
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ *        do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
+    int i;
+    for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0;
+}
 
 /*! ZSTD_copyCCtx() :
 *   Duplicate an existing context `srcCCtx` into another one `dstCCtx`.

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 93220f5c9ccc5fada152de0858f669367846fdac..b060b73f483ab29d6d50f40313a8e129d3414576 100644 (file)
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -176,10 +176,10 @@ typedef struct {
     ZSTD_CCtx* cctx;
     buffer_t src;
     const void* srcStart;
-    size_t srcSize;
+    size_t   srcSize;
     buffer_t dstBuff;
-    size_t cSize;
-    size_t dstFlushed;
+    size_t   cSize;
+    size_t   dstFlushed;
     unsigned long long fullFrameSize;
     unsigned firstChunk;
     unsigned lastChunk;
@@ -196,9 +196,10 @@ void ZSTDMT_compressChunk(void* jobDescription)
     buffer_t const dstBuff = job->dstBuff;
     size_t const initError = ZSTD_compressBegin_advanced(job->cctx, NULL, 0, job->params, job->fullFrameSize);
     if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
-    if (!job->firstChunk) {
-        size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, job->srcStart, 0);   /* flush frame header */
+    if (!job->firstChunk) {  /* flush frame header */
+        size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, job->srcStart, 0);
         if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
+        ZSTD_invalidateRepCodes(job->cctx);
     }
 
     DEBUGLOG(3, "Compressing : ");