From: Yann Collet <cyan@fb.com>
Date: Fri, 2 Jun 2017 01:44:06 +0000 (-0700)
Subject: added support for multithreading parameters
X-Git-Tag: v1.3.0~1^2~17^2~65
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c35e535002bf6b178f6c5ff53ce9b348fa8de427;p=thirdparty%2Fzstd.git

added support for multithreading parameters
---

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 7924968e4..8f7c200b2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -157,6 +157,7 @@ struct ZSTD_CCtx_s {
     U32    frameEnded;
 
     /* Multi-threading */
+    U32 nbThreads;
     ZSTDMT_CCtx* mtctx;
 };
 
@@ -353,9 +354,33 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
     case ZSTD_p_forceMaxWindow :  /* Force back-references to remain < windowSize,
                                    * even when referencing into Dictionary content
                                    * default : 0 when using a CDict, 1 when using a Prefix */
-            cctx->forceWindow = value>0;
-            cctx->loadedDictEnd = 0;
-            return 0;
+        cctx->forceWindow = value>0;
+        cctx->loadedDictEnd = 0;
+        return 0;
+
+    case ZSTD_p_nbThreads:
+        if (value==0) return 0;
+#ifndef ZSTD_MULTITHREAD
+        if (value > 1) return ERROR(compressionParameter_unsupported);
+#endif
+        if ((value>1) && (cctx->nbThreads != value)) {
+            ZSTDMT_freeCCtx(cctx->mtctx);
+            cctx->nbThreads = value;
+            cctx->mtctx = ZSTDMT_createCCtx(value);
+            if (cctx->mtctx == NULL) return ERROR(memory_allocation);
+        }
+        cctx->nbThreads = 1;
+        return 0;
+
+    case ZSTDMT_p_jobSize:
+        if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
+        assert(cctx->mtctx != NULL);
+        return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_sectionSize, value);
+
+    case ZSTDMT_p_overlapSizeLog:
+        if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
+        assert(cctx->mtctx != NULL);
+        return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
 
     case ZSTD_p_rawContentDict :  /* load dictionary in "content-only" mode (no header analysis) (default:0) */
         cctx->forceRawDict = value>0;
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 57c0360b0..0d6014608 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -439,7 +439,7 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter,
         mtctx->sectionSize = value;
         return 0;
     case ZSTDMT_p_overlapSectionLog :
-    DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
+        DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
         mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value;
         return 0;
     default :
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index ff7f21687..336215995 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -33,7 +33,7 @@ ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
 
 ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
 ZSTDLIB_API size_t ZSTDMT_estimateCCtxSize(ZSTD_compressionParameters cParams,
-                                           unsigned nbThreads);
+                                           unsigned nbThreads);   /* not ready yet */
 
 
 /* ===   Simple buffer-to-butter one-pass function   === */
diff --git a/lib/zstd.h b/lib/zstd.h
index 1fc35fc55..c71ed493e 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -491,7 +491,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  *  These functions make it possible to estimate memory usage
  *  of a future target object, before its allocation,
  *  given a set of parameters, which vary depending on target object.
- *  The objective is to guide decision before allocation. */
+ *  The objective is to guide decision before allocation.
+ *  Note : CCtx estimation is only correct for single-threaded compression */
 ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
 ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
 
@@ -695,17 +696,17 @@ typedef enum {
     ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
                              /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
 
-#if 0
-    /* multi-threading parameters (not ready yet !) */
+    /* multi-threading parameters */
     ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
-                              * More threads improve speed, but increases also memory usage */
-    ZSTDMT_p_jobSize,        /* Size of a compression job. Each job is compressed in parallel.
+                              * More threads improve speed, but also increase memory usage.
+                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
+                              * Special: value 0 means "do not change nbThreads" */
+    ZSTDMT_p_jobSize,        /* Size of a compression job. Each compression job is completed in parallel.
                               * 0 means default, which is dynamically determined based on compression parameters.
                               * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
                               * The minimum size is automatically and transparently enforced */
     ZSTDMT_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
                               * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-#endif
 
     /* advanced parameters - may not remain available after API update */
     ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,