From: Yann Collet Date: Wed, 14 Nov 2018 02:09:03 +0000 (-0800) Subject: added constant ZSTD_WINDOWLOG_LIMIT_DEFAULT X-Git-Tag: v1.3.8~33^2~37 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d7e10a774a7ade1deeb8b22e74a51e49edb3a6f0;p=thirdparty%2Fzstd.git added constant ZSTD_WINDOWLOG_LIMIT_DEFAULT answering #1407. Also : removed obsolete function ZSTD_setDStreamParameter() which could only be used with one parameter (DStream_p_maxWindowSize). Now replaced by ZSTD_DCtx_setWindowSize() (which exists since a few revisions) --- diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 7963bca19..f8a189293 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -20,17 +20,18 @@
  • Streaming decompression - HowTo
  • ADVANCED AND EXPERIMENTAL FUNCTIONS
  • Candidate API for promotion into stable
    • -
  • Purely experimental API
    • -
  • Frame size functions
    • -
  • Memory management
    • -
  • Advanced compression functions
    • -
  • Advanced decompression functions
    • -
  • Advanced streaming functions
    • -
  • Buffer-less and synchronous inner streaming functions
    • -
  • Buffer-less streaming compression (synchronous mode)
    • -
  • Buffer-less streaming decompression (synchronous mode)
    • -
  • New advanced API (experimental)
    • -
  • Block level API
    • +
  • Memory management
    • +
  • Purely experimental API
    • +
  • Frame size functions
    • +
  • Memory management
    • +
  • Advanced compression functions
    • +
  • Advanced decompression functions
    • +
  • Advanced streaming functions
    • +
  • Buffer-less and synchronous inner streaming functions
    • +
  • Buffer-less streaming compression (synchronous mode)
    • +
  • Buffer-less streaming decompression (synchronous mode)
    • +
  • New advanced API (experimental)
    • +
  • Block level API

    • Introduction

    @@ -357,10 +358,23 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
 
    int ZSTD_minCLevel(void);  /*!< minimum negative compression level allowed */
 

    
-
    Purely experimental API
    - The following symbols and constants are not planned
- to join "stable API" status anytime soon.
- Some of them might even be removed in the future.
+
    Memory management
    
+
+
    size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
      These functions give the current memory usage of selected object.
+  Object memory usage can evolve when re-used. 
+

    
+
+
    Purely experimental API
    + The following symbols and constants
+ are not planned to join "stable API" status anytime soon.
+ Some will never reach "stable", and remain in the static_only section.
+ Some of them might be removed in the future.
  
 
    
 
@@ -407,7 +421,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     ZSTD_dlm_byRef = 1,   /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
 } ZSTD_dictLoadMethod_e;
 

    
-
    Frame size functions
    
+
    Frame size functions
    
 
 
    size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
 
      `src` should point to the start of a ZSTD encoded frame or skippable frame
@@ -446,17 +460,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
            or an error code (if srcSize is too small) 
 

    
 
-
    Memory management
    
-
-
    size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
-size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
-size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
-size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
-
      These functions give the current memory usage of selected object.
-  Object memory usage can evolve when re-used. 
-

    
+
    Memory management
    
 
 
    size_t ZSTD_estimateCCtxSize(int compressionLevel);
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -536,7 +540,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
  
 

    
 
-
    Advanced compression functions
    
+
    Advanced compression functions
    
 
 
    ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
      Create a digested dictionary for compression
@@ -578,7 +582,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
 
       Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters 
 

    
 
-
    Advanced decompression functions
    
+
    Advanced decompression functions
    
 
 
    unsigned ZSTD_isFrame(const void* buffer, size_t size);
 
      Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -618,7 +622,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
   When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. 
 

    
 
-
    Advanced streaming functions
    
+
    Advanced streaming functions
    
 
 
    Advanced Streaming compression functions
    size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
@@ -658,25 +662,23 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
   + there is no active job (could be checked with ZSTD_frameProgression()), or
   + oldest job is still actively compressing data,
     but everything it has produced has also been flushed so far,
-    therefore flushing speed is currently limited by production speed of oldest job
-    irrespective of the speed of concurrent newer jobs.
+    therefore flush speed is limited by production speed of oldest job
+    irrespective of the speed of concurrent (and newer) jobs.
  
 

    
 
-
    Advanced Streaming decompression functions
    typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
-size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);   /* obsolete : this API will be removed in a future version */
-size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
+
    Advanced Streaming decompression functions
    size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict is referenced, it must outlive decompression session */
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
 

    
-
    Buffer-less and synchronous inner streaming functions
    +
    Buffer-less and synchronous inner streaming functions
       This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with several restrictions, documented below.
   Prefer normal streaming API for an easier experience.
  
 
    
 
-
    Buffer-less streaming compression (synchronous mode)
    +
    Buffer-less streaming compression (synchronous mode)
       A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -712,7 +714,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 

    
-
    Buffer-less streaming decompression (synchronous mode)
    +
    Buffer-less streaming decompression (synchronous mode)
       A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
@@ -803,7 +805,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
 

    
 
    typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 

    
-
    New advanced API (experimental)
    
+
    New advanced API (experimental)
    
 
 
    typedef enum {
     /* Opened question : should we have a format ZSTD_f_auto ?
@@ -1248,7 +1250,7 @@ size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx,
 
      Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
   This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
   This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
-  By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
+  By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
  @return : 0, or an error code (which can be tested using ZSTD_isError()).
  
 

    
@@ -1299,7 +1301,7 @@ size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx,
  
 

    
 
-
    Block level API
    
+
    Block level API
    
 
 
        Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     User will have to take in charge required information to regenerate data, such as compressed and content sizes.
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index e6286548d..edeb74b9c 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -78,7 +78,6 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
 #define BIT0   1
 
 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
-#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
 static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
 static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
 
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index 21fba4d59..c8547f8fb 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -21,7 +21,7 @@ extern "C" {
 *  Long distance matching
 ***************************************/
 
-#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
+#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
 
 /**
  * ZSTD_ldm_generateSequences():
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 6bd2e3ef3..76f81a37c 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -37,7 +37,7 @@
  *  It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
  */
 #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-#  define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
+#  define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1)
 #endif
 
 /*!
@@ -1233,20 +1233,6 @@ size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
     return ZSTD_FRAMEHEADERSIZE_PREFIX;
 }
 
-size_t ZSTD_setDStreamParameter(ZSTD_DStream* dctx,
-                                ZSTD_DStreamParameter_e paramType, unsigned paramValue)
-{
-    if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
-    switch(paramType)
-    {
-        default : return ERROR(parameter_unsupported);
-        case DStream_p_maxWindowSize :
-            DEBUGLOG(4, "setting maxWindowSize = %u KB", paramValue >> 10);
-            dctx->maxWindowSize = paramValue ? paramValue : (U32)(-1);
-            break;
-    }
-    return 0;
-}
 
 size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
 {
@@ -1296,7 +1282,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize)
 
 size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
 {
-    U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;   /* note : should be user-selectable */
+    U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;   /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
     ZSTD_frameHeader zfh;
     size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
     if (ZSTD_isError(err)) return err;
diff --git a/lib/dll/libzstd.def b/lib/dll/libzstd.def
index 51d0c1925..668c4b1c2 100644
--- a/lib/dll/libzstd.def
+++ b/lib/dll/libzstd.def
@@ -78,7 +78,6 @@ EXPORTS
     ZSTD_nextSrcSizeToDecompress
     ZSTD_resetCStream
     ZSTD_resetDStream
-    ZSTD_setDStreamParameter
     ZSTD_sizeof_CCtx
     ZSTD_sizeof_CDict
     ZSTD_sizeof_CStream
diff --git a/lib/zstd.h b/lib/zstd.h
index e66738fe5..78b13cb19 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -422,8 +422,14 @@ ZSTDLIB_API int ZSTD_minCLevel(void);  /*!< minimum negative compression level a
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50    /* all 16 values, from 0x184D2A50 to 0x184D2A5F, are understood as skippable frames */
 #define ZSTD_MAGIC_SKIPPABLE_MASK   0xFFFFFFF0
 
+/* note : should this limit be smaller ? like 23 (8 MB) as recommended in the spec ?
+ * also: could it be different from zstd cli default limit, which is designed to match LDM default (27) ? */
+#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27   /* by default, the streaming decoder will refuse any frame
+                                           * requiring larger than (1<