From: Yann Collet <cyan@fb.com>
Date: Thu, 19 Jan 2017 18:32:55 +0000 (-0800)
Subject: fixed Multi-threaded compression
X-Git-Tag: v1.1.3^2~19^2~33
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=32dfae6f9841871d88eadcd27a970efab71feb28;p=thirdparty%2Fzstd.git

fixed Multi-threaded compression

MT compression generates a single frame.
Multi-threading operates by breaking the frames into independent sections.
But from a decoder perspective, there is no difference :
it's just a suite of blocks.

Problem is, decoder preserves repCodes from previous block to start decoding next block.
This is also valid between sections, since they are no different than changing block.

Previous version would incorrectly initialize repcodes to their default value at the beginning of each section.
When using them, there was a mismatch between encoder (default values) and decoder (values from previous block).

This change ensures that repcodes won't be used at the beginning of a new section.
It works by setting them to 0.
This only works with regular (single segment) variants : extDict variants will fail !
Fortunately, sections beyond the 1st one belong to this category.

To be checked : btopt strategy.
This change was only validated from fast to btlazy2 strategies.
---

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 96e057758..4b56ce1a2 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -267,4 +267,13 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
 }
 
 
+/* hidden functions */
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ *        do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
+
+
 #endif   /* ZSTD_CCOMMON_H_MODULE */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index d4800dce1..84a4a0216 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -317,6 +317,14 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     }
 }
 
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ *        do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
+    int i;
+    for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0;
+}
 
 /*! ZSTD_copyCCtx() :
 *   Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 93220f5c9..b060b73f4 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -176,10 +176,10 @@ typedef struct {
     ZSTD_CCtx* cctx;
     buffer_t src;
     const void* srcStart;
-    size_t srcSize;
+    size_t   srcSize;
     buffer_t dstBuff;
-    size_t cSize;
-    size_t dstFlushed;
+    size_t   cSize;
+    size_t   dstFlushed;
     unsigned long long fullFrameSize;
     unsigned firstChunk;
     unsigned lastChunk;
@@ -196,9 +196,10 @@ void ZSTDMT_compressChunk(void* jobDescription)
     buffer_t const dstBuff = job->dstBuff;
     size_t const initError = ZSTD_compressBegin_advanced(job->cctx, NULL, 0, job->params, job->fullFrameSize);
     if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
-    if (!job->firstChunk) {
-        size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, job->srcStart, 0);   /* flush frame header */
+    if (!job->firstChunk) {  /* flush frame header */
+        size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, job->srcStart, 0);
         if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
+        ZSTD_invalidateRepCodes(job->cctx);
     }
 
     DEBUGLOG(3, "Compressing : ");