updated ZSTD_estimate?DictSize() to pass parameter byReference

resulting ?Dict object is smaller when created byReference.
Seems better than a documentation note.

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 72da136503e0970807b74a9f6f21577112f40d61..f6a09ef0cfbdbfecf0cf771ac31370be1ba0a881 100644 (file)
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -412,9 +412,9 @@ size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
          In this case, get additional size by using ZSTD_estimate?DictSize 
 </p></pre><BR>
 
-<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
-size_t ZSTD_estimateDDictSize(size_t dictSize);
-</b><p>  Note : if dictionary is created "byReference", reduce estimation by dictSize 
+<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
+size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
+</b><p>  Note : dictionary created "byReference" are smaller 
 </p></pre><BR>
 
 <a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
@@ -718,6 +718,21 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
 </b><p>  Create a ZSTD_DDict using external alloc and free, optionally by reference 
 </p></pre><BR>
 
+<pre><b>ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
+                                 const void* dict, size_t dictSize,
+                                 unsigned byReference);
+</b><p>  Generate a digested dictionary in provided memory area.
+  workspace: The memory area to emplace the dictionary into.
+             Provided pointer must 8-bytes aligned.
+             It must outlive dictionary usage.
+  workspaceSize: Use ZSTD_estimateDDictSize()
+                 to determine how large workspace must be.
+ @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
+  Note : there is no corresponding "free" function.
+         Since workspace was allocated externally, it must be freed externally.
+ 
+</p></pre><BR>
+
 <pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
 </b><p>  Provides the dictID stored within dictionary.
   if @return == 0, the dictionary is not conformant with Zstandard specification.

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 58ed98e9bd6fa91a5227240780d3eb515b77c5f2..aa9f64c5c8176526493c431937217624f40fdd2a 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3237,10 +3237,11 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
 
 /*! ZSTD_estimateCDictSize() :
  *  Estimate amount of memory that will be needed to create a dictionary with following arguments */
-size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
+size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
 {
     cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
-    return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams);
+    return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize(cParams)
+           + (byReference ? 0 : dictSize);
 }
 
 size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 2db493e51b76be2967d2fd9f3fe90d56075ac764..8b13811840192dde89e51069c1823072863d9a3a 100644 (file)
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2053,10 +2053,10 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
 
 /*! ZSTD_estimateDDictSize() :
  *  Estimate amount of memory that will be needed to create a dictionary for decompression.
- *  Note : if dictionary is created "byReference", reduce this amount by dictSize */
-size_t ZSTD_estimateDDictSize(size_t dictSize)
+ *  Note : dictionary created "byReference" are smaller */
+size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference)
 {
-    return dictSize + sizeof(ZSTD_DDict);
+    return sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
 }
 
 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)

diff --git a/lib/zstd.h b/lib/zstd.h
index bb043e486fa94eef46a034369eb70cfe5e47390a..de3b2c30dc0c91a47591748bf86da84dec59df53 100644 (file)
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -345,8 +345,6 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 #endif  /* ZSTD_H_235446 */
 
 
-#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
-#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
 
 /****************************************************************************************
  * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
@@ -356,6 +354,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
  * Use them only in association with static linking.
  * ***************************************************************************************/
 
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+
 /* --- Constants ---*/
 #define ZSTD_MAGICNUMBER            0xFD2FB528   /* >= v0.8.0 */
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
@@ -500,9 +501,9 @@ ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
 ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
 /*! ZSTD_estimate?DictSize() :
- *  Note : if dictionary is created "byReference", reduce estimation by dictSize */
-ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
-ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
+ *  Note : dictionary created "byReference" are smaller */
+ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
+ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
 
 
 /***************************************
@@ -842,6 +843,21 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
                                                   unsigned byReference, ZSTD_customMem customMem);
 
+/*! ZSTD_initStaticDDict() :
+ *  Generate a digested dictionary in provided memory area.
+ *  workspace: The memory area to emplace the dictionary into.
+ *             Provided pointer must 8-bytes aligned.
+ *             It must outlive dictionary usage.
+ *  workspaceSize: Use ZSTD_estimateDDictSize()
+ *                 to determine how large workspace must be.
+ * @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
+ *  Note : there is no corresponding "free" function.
+ *         Since workspace was allocated externally, it must be freed externally.
+ */
+ZSTDLIB_API ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
+                                             const void* dict, size_t dictSize,
+                                             unsigned byReference);
+
 /*! ZSTD_getDictID_fromDict() :
  *  Provides the dictID stored within dictionary.
  *  if @return == 0, the dictionary is not conformant with Zstandard specification.

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index b2de6ec69cf806607d3a7a75faf15594df8d3195..83b95974f1800b3d490f0ce2e6e43dab51099960 100644 (file)
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -490,14 +490,15 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
-            size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
+            size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize, 1);
             DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
         }
 
         DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
-            ZSTD_customMem customMem = { NULL, NULL, NULL };
-            ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
+            ZSTD_customMem const customMem = { NULL, NULL, NULL };
+            ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
+                                    1 /* by Referece */, cParams, customMem);
             cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
                                                  CNBuffer, CNBuffSize, cdict);
             ZSTD_freeCDict(cdict);

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index e9e17031c7cf0272c01d4493c538137e7951c643..0f73b8cd4a24413de3cf39f79dec38841875a39e 100644 (file)
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -209,7 +209,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
         size_t const s = ZSTD_estimateCStreamSize(cParams)
-                       + ZSTD_estimateCDictSize(cParams, dictSize);  /* uses ZSTD_initCStream_usingDict() */
+                        /* uses ZSTD_initCStream_usingDict() */
+                       + ZSTD_estimateCDictSize(cParams, dictSize, 0);
             if (ZSTD_isError(s)) goto _output_error;
             DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
     }
@@ -281,7 +282,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         if (gfhError!=0) goto _output_error;
         DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
         {   size_t const s = ZSTD_estimateDStreamSize(fhi)
-                           + ZSTD_estimateDDictSize(dictSize);  /* uses ZSTD_initDStream_usingDict() */
+                            /* uses ZSTD_initDStream_usingDict() */
+                           + ZSTD_estimateDDictSize(dictSize, 0);
             if (ZSTD_isError(s)) goto _output_error;
             DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
     }   }