From: Yann Collet Date: Tue, 9 May 2017 00:51:49 +0000 (-0700) Subject: added ZSTD_estimateCDictSize() and ZSTD_estimateDDictSize() X-Git-Tag: v1.3.0~1^2~46^2~10 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a1d6704d7f63fb9ad177a8364a40385206d21b53;p=thirdparty%2Fzstd.git added ZSTD_estimateCDictSize() and ZSTD_estimateDDictSize() it complements ZSTD_estimateCCtxSize() for the special case of ZSTD_initCStream_usingDict() --- diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index d70915458..9cfb67100 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -372,15 +372,17 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v

Advanced compression functions


 
-
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
-
  Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
-  `frameContentSize` is an optional parameter, provide `0` if unknown 
-


-
 
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 
  Create a ZSTD compression context using external alloc and free functions 
 


 
+
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
+
  Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
+  Set streaming to 1 if the CCtx will be used for streaming (CStream).
+  Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
+         Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size 
+


+
 
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 
  amount of used memory is variable, depending primarily on compression level 
 


@@ -406,6 +408,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
  Create a ZSTD_CDict using external alloc and free, and customized compression parameters 
 


 
+
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
+
  Estimate amount of memory that will be needed to create a dictionary with following arguments
+  Note : if dictionary is created "byReference", reduce this amount by dictSize 
+


+
 
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
 
  Gives the amount of memory used by a given ZSTD_sizeof_CDict 
 


@@ -476,6 +483,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
  Create a ZSTD_DDict using external alloc and free, optionally by reference 
 


 
+
size_t ZSTD_estimateDDictSize(size_t dictSize);
+
  Estimate amount of memory that will be needed to create a dictionary for decompression.
+  Note : if dictionary is created "byReference", reduce this amount by dictSize 
+


+
 
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
  Gives the amount of memory used by a given ZSTD_DDict 
 


@@ -509,7 +521,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);   /**< same as ZSTD_sizeof_CCtx */
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 748aff930..c5687b21c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2984,6 +2984,14 @@ struct ZSTD_CDict_s {
     ZSTD_CCtx* refContext;
 };  /* typedef'd tp ZSTD_CDict within "zstd.h" */
 
+/*! ZSTD_estimateCDictSize() :
+ *  Estimate amount of memory that will be needed to create a dictionary with following arguments */
+size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
+{
+    cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
+    return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams, 0);
+}
+
 size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
 {
     if (cdict==NULL) return 0;   /* support sizeof on NULL */
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 910f9ab78..88488b4b0 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1969,6 +1969,14 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
     }
 }
 
+/*! ZSTD_estimateDDictSize() :
+ *  Estimate amount of memory that will be needed to create a dictionary for decompression.
+ *  Note : if dictionary is created "byReference", reduce this amount by dictSize */
+size_t ZSTD_estimateDDictSize(size_t dictSize)
+{
+    return dictSize + sizeof(ZSTD_DDict);
+}
+
 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;   /* support sizeof on NULL */
diff --git a/lib/zstd.h b/lib/zstd.h
index e8614334b..270943b30 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -458,17 +458,17 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
 /***************************************
 *  Advanced compression functions
 ***************************************/
-/*! ZSTD_estimateCCtxSize() :
- *  Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
- *  Set streaming to 1 if the CCtx will be used for streaming (CStream)
- *  Note : this function is currently unable to estimate additional memory allocation needed to create an internal CDict
- *         which can only happen when starting with ZSTD_initCStream_usingDict() */
-ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
-
 /*! ZSTD_createCCtx_advanced() :
  *  Create a ZSTD compression context using external alloc and free functions */
 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 
+/*! ZSTD_estimateCCtxSize() :
+ *  Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
+ *  Set streaming to 1 if the CCtx will be used for streaming (CStream).
+ *  Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
+ *         Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size */
+ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
+
 /*! ZSTD_sizeofCCtx() :
  *  amount of used memory is variable, depending primarily on compression level */
 ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
@@ -493,6 +493,11 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
                                                   ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
 
+/*! ZSTD_estimateCDictSize() :
+ *  Estimate amount of memory that will be needed to create a dictionary with following arguments
+ *  Note : if dictionary is created "byReference", reduce this amount by dictSize */
+ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
+
 /*! ZSTD_sizeof_CDict() :
  *  Gives the amount of memory used by a given ZSTD_sizeof_CDict */
 ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
@@ -564,6 +569,11 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
                                                   unsigned byReference, ZSTD_customMem customMem);
 
+/*! ZSTD_estimateDDictSize() :
+ *  Estimate amount of memory that will be needed to create a dictionary for decompression.
+ *  Note : if dictionary is created "byReference", reduce this amount by dictSize */
+ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
+
 /*! ZSTD_sizeof_DDict() :
  *  Gives the amount of memory used by a given ZSTD_DDict */
 ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a9dcf12e0..7ffcff26e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -404,6 +404,12 @@ static int basicUnitTests(U32 seed, double compressibility)
                   if (r != CNBuffSize) goto _output_error);
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
+        {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
+            size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
+            DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
+        }
+
         DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
             ZSTD_customMem customMem = { NULL, NULL, NULL };