added ZSTD_estimateCDictSize() and ZSTD_estimateDDictSize()

it complements ZSTD_estimateCCtxSize()
for the special case of ZSTD_initCStream_usingDict()

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index d709154588d761b036ef4fb563007a49f0234809..9cfb6710008c502f2b4085d3709c66b20162e117 100644 (file)
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -372,15 +372,17 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
 <a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
 
-<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
-</b><p>  Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
-  `frameContentSize` is an optional parameter, provide `0` if unknown 
-</p></pre><BR>
-
 <pre><b>ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 </b><p>  Create a ZSTD compression context using external alloc and free functions 
 </p></pre><BR>
 
+<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
+</b><p>  Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
+  Set streaming to 1 if the CCtx will be used for streaming (CStream).
+  Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
+         Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size 
+</p></pre><BR>
+
 <pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 </b><p>  amount of used memory is variable, depending primarily on compression level 
 </p></pre><BR>
@@ -406,6 +408,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </b><p>  Create a ZSTD_CDict using external alloc and free, and customized compression parameters 
 </p></pre><BR>
 
+<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
+</b><p>  Estimate amount of memory that will be needed to create a dictionary with following arguments
+  Note : if dictionary is created "byReference", reduce this amount by dictSize 
+</p></pre><BR>
+
 <pre><b>size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
 </b><p>  Gives the amount of memory used by a given ZSTD_sizeof_CDict 
 </p></pre><BR>
@@ -476,6 +483,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </b><p>  Create a ZSTD_DDict using external alloc and free, optionally by reference 
 </p></pre><BR>
 
+<pre><b>size_t ZSTD_estimateDDictSize(size_t dictSize);
+</b><p>  Estimate amount of memory that will be needed to create a dictionary for decompression.
+  Note : if dictionary is created "byReference", reduce this amount by dictSize 
+</p></pre><BR>
+
 <pre><b>size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 </b><p>  Gives the amount of memory used by a given ZSTD_DDict 
 </p></pre><BR>
@@ -509,7 +521,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);   </b>/**< same as ZSTD_sizeof_CCtx */<b>
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */<b>
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */<b>
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 748aff930172c86de705a7613cbb1add83bd59f7..c5687b21c193ae3206526849f317539fa9a06b23 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2984,6 +2984,14 @@ struct ZSTD_CDict_s {
     ZSTD_CCtx* refContext;
 };  /* typedef'd tp ZSTD_CDict within "zstd.h" */
 
+/*! ZSTD_estimateCDictSize() :
+ *  Estimate amount of memory that will be needed to create a dictionary with following arguments */
+size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
+{
+    cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
+    return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams, 0);
+}
+
 size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
 {
     if (cdict==NULL) return 0;   /* support sizeof on NULL */

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 910f9ab783c31765890735e8424ce054da5c30dc..88488b4b0525208f73803ebd59cb5bb36799591a 100644 (file)
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1969,6 +1969,14 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
     }
 }
 
+/*! ZSTD_estimateDDictSize() :
+ *  Estimate amount of memory that will be needed to create a dictionary for decompression.
+ *  Note : if dictionary is created "byReference", reduce this amount by dictSize */
+size_t ZSTD_estimateDDictSize(size_t dictSize)
+{
+    return dictSize + sizeof(ZSTD_DDict);
+}
+
 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;   /* support sizeof on NULL */

diff --git a/lib/zstd.h b/lib/zstd.h
index e8614334be267ff759e99e03fdb3ab8490c30eed..270943b308be0224877de1f9ef866a186567208e 100644 (file)
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -458,17 +458,17 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
 /***************************************
 *  Advanced compression functions
 ***************************************/
-/*! ZSTD_estimateCCtxSize() :
- *  Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
- *  Set streaming to 1 if the CCtx will be used for streaming (CStream)
- *  Note : this function is currently unable to estimate additional memory allocation needed to create an internal CDict
- *         which can only happen when starting with ZSTD_initCStream_usingDict() */
-ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
-
 /*! ZSTD_createCCtx_advanced() :
  *  Create a ZSTD compression context using external alloc and free functions */
 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 
+/*! ZSTD_estimateCCtxSize() :
+ *  Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
+ *  Set streaming to 1 if the CCtx will be used for streaming (CStream).
+ *  Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
+ *         Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size */
+ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
+
 /*! ZSTD_sizeofCCtx() :
  *  amount of used memory is variable, depending primarily on compression level */
 ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
@@ -493,6 +493,11 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
                                                   ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
 
+/*! ZSTD_estimateCDictSize() :
+ *  Estimate amount of memory that will be needed to create a dictionary with following arguments
+ *  Note : if dictionary is created "byReference", reduce this amount by dictSize */
+ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
+
 /*! ZSTD_sizeof_CDict() :
  *  Gives the amount of memory used by a given ZSTD_sizeof_CDict */
 ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
@@ -564,6 +569,11 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
                                                   unsigned byReference, ZSTD_customMem customMem);
 
+/*! ZSTD_estimateDDictSize() :
+ *  Estimate amount of memory that will be needed to create a dictionary for decompression.
+ *  Note : if dictionary is created "byReference", reduce this amount by dictSize */
+ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
+
 /*! ZSTD_sizeof_DDict() :
  *  Gives the amount of memory used by a given ZSTD_DDict */
 ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a9dcf12e0702626540ccd6ec4de1efb4dd1dfd07..7ffcff26e13ebfe95a6adeba649a9ad15c786598 100644 (file)
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -404,6 +404,12 @@ static int basicUnitTests(U32 seed, double compressibility)
                   if (r != CNBuffSize) goto _output_error);
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
+        {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
+            size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
+            DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
+        }
+
         DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
             ZSTD_customMem customMem = { NULL, NULL, NULL };