From f77f634d41149c3e5754ebfe4d5cf3a5f138c843 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 24 Feb 2024 01:28:17 -0800 Subject: [PATCH] update API documentation --- doc/zstd_manual.html | 67 +++++++++++++++++++++----------- lib/decompress/zstd_decompress.c | 8 ++-- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index d72eacc34..c81627d87 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -156,7 +156,7 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); * for example to size a static array on stack. * Will produce constant value 0 if srcSize too large. */ -#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U) +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U) #define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ /* ZSTD_isError() : @@ -513,6 +513,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ * ZSTD_d_forceIgnoreChecksum * ZSTD_d_refMultipleDDicts * ZSTD_d_disableHuffmanAssembly + * ZSTD_d_maxBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly */ @@ -520,7 +521,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ ZSTD_d_experimentalParam2=1001, ZSTD_d_experimentalParam3=1002, ZSTD_d_experimentalParam4=1003, - ZSTD_d_experimentalParam5=1004 + ZSTD_d_experimentalParam5=1004, + ZSTD_d_experimentalParam6=1005 } ZSTD_dParameter;
@@ -1386,58 +1388,61 @@ ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,

Memory management


 
-
ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
 
  These functions make it possible to estimate memory usage
   of a future {D,C}Ctx, before its creation.
+  This is useful in combination with ZSTD_initStatic(),
+  which makes it possible to employ a static buffer for ZSTD_CCtx* state.
 
   ZSTD_estimateCCtxSize() will provide a memory budget large enough
-  for any compression level up to selected one.
-  Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
-         does not include space for a window buffer.
-         Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
+  to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2()
+  associated with any compression level up to max specified one.
   The estimate will assume the input may be arbitrarily large,
   which is the worst case.
 
+  Note that the size estimation is specific for one-shot compression,
+  it is not valid for streaming (see ZSTD_estimateCStreamSize*())
+  nor other potential ways of using a ZSTD_CCtx* state.
+
   When srcSize can be bound by a known and rather "small" value,
-  this fact can be used to provide a tighter estimation
-  because the CCtx compression context will need less memory.
-  This tighter estimation can be provided by more advanced functions
+  this knowledge can be used to provide a tighter budget estimation
+  because the ZSTD_CCtx* state will need less memory for small inputs.
+  This tighter estimation can be provided by employing more advanced functions
   ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
   and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
   Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
 
   Note : only single-threaded compression is supported.
   ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
-
-  Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
-  Size estimates assume that no external sequence producer is registered.
  
 


 
-
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
-
  ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
-  It will also consider src size to be arbitrarily "large", which is worst case.
+
  ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression
+  using any compression level up to the max specified one.
+  It will also consider src size to be arbitrarily "large", which is a worst case scenario.
   If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
   ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
   ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
   Note : CStream size estimation is only correct for single-threaded compression.
-  ZSTD_DStream memory budget depends on window Size.
+  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+  Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+  Size estimates assume that no external sequence producer is registered.
+
+  ZSTD_DStream memory budget depends on frame's window Size.
   This information can be passed manually, using ZSTD_estimateDStreamSize,
   or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
+  Any frame requesting a window size larger than max specified one will be rejected.
   Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
          an internal ?Dict will be created, which additional size is not estimated here.
          In this case, get total size by adding ZSTD_estimate?DictSize
-  Note 2 : only single-threaded compression is supported.
-  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
-  Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
-  Size estimates assume that no external sequence producer is registered.
  
 


 
@@ -1926,7 +1931,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 ZSTD_registerSequenceProducer(
   ZSTD_CCtx* cctx,
   void* sequenceProducerState,
-  ZSTD_sequenceProducer_F* sequenceProducer
+  ZSTD_sequenceProducer_F sequenceProducer
 );
 
 Instruct zstd to use a block-level external sequence producer function.
 
@@ -1948,6 +1953,22 @@ ZSTD_registerSequenceProducer(
  calling this function. 
 


 
+
ZSTDLIB_STATIC_API void
+ZSTD_CCtxParams_registerSequenceProducer(
+  ZSTD_CCtx_params* params,
+  void* sequenceProducerState,
+  ZSTD_sequenceProducer_F sequenceProducer
+);
+
 Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params.
+ This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(),
+ which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx().
+
+ If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx()
+ is required, then this function is for you. Otherwise, you probably don't need it.
+
+ See tests/zstreamtest.c for example usage. 
+


+
 
Buffer-less and synchronous inner streaming functions (DEPRECATED)
   This API is deprecated, and will be removed in a future version.
   It allows streaming (de)compression with user allocated buffers.
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 0d17b5e07..173059085 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -18,7 +18,6 @@
  * on stack (0), or into heap (1, default; requires malloc()).
  * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.
  */
-#include "error_private.h"
 #ifndef ZSTD_HEAPMODE
 #  define ZSTD_HEAPMODE 1
 #endif
@@ -56,18 +55,19 @@
 /*-*******************************************************
 *  Dependencies
 *********************************************************/
-#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
 #include "../common/zstd_deps.h"   /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
+#include "../common/error_private.h"
+#include "../common/zstd_internal.h"  /* blockProperties_t */
 #include "../common/mem.h"         /* low level memory routines */
+#include "../common/bits.h"  /* ZSTD_highbit32 */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
 #include "../common/huf.h"
 #include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
-#include "../common/zstd_internal.h"  /* blockProperties_t */
 #include "zstd_decompress_internal.h"   /* ZSTD_DCtx */
 #include "zstd_ddict.h"  /* ZSTD_DDictDictContent */
 #include "zstd_decompress_block.h"   /* ZSTD_decompressBlock_internal */
-#include "../common/bits.h"  /* ZSTD_highbit32 */
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
 #  include "../legacy/zstd_legacy.h"
-- 
2.47.2