reduced memory usage

by avoiding to duplicate in memory
a dictionary that was passed by reference.

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 76c6d3f2c7fbf956c791a78d63233e03d1653cdd..bcd06b7f1ac1c929d665368904ac62aeff86909f 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1354,10 +1354,12 @@ size_t ZSTD_CCtx_refPrefix_advanced(
     RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                     "Can't ref a prefix when ctx not in init stage.");
     ZSTD_clearAllDicts(cctx);
-    if (prefix != NULL && prefixSize > 0) {
+    if (prefixSize > 0) {
+        RETURN_ERROR_IF(prefix == NULL, dictionary_wrong, "Invalid prefix pointer");
         cctx->prefixDict.dict = prefix;
         cctx->prefixDict.dictSize = prefixSize;
         cctx->prefixDict.dictContentType = dictContentType;
+        cctx->prefixDict.loadMethod = ZSTD_dlm_byRef;
     }
     return 0;
 }
@@ -3066,7 +3068,7 @@ ZSTD_entropyCompressSeqStore(
 /* ZSTD_selectBlockCompressor() :
  * Not static, but internal use only (used by long distance matcher)
  * assumption : strat is a valid strategy */
-ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode)
+ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e useRowMatchFinder, ZSTD_DictMode_e dictMode)
 {
     static const ZSTD_BlockCompressor_f blockCompressor[4][ZSTD_STRATEGY_MAX+1] = {
         { ZSTD_compressBlock_fast  /* default for 0 */,
@@ -3298,7 +3300,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
     }
 
     /* select and store sequences */
-    {   ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms);
+    {   ZSTD_DictMode_e const dictMode = ZSTD_matchState_dictMode(ms);
         size_t lastLLSize;
         {   int i;
             for (i = 0; i < ZSTD_REP_NUM; ++i)
@@ -6351,7 +6353,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
                                              size_t inSize)
 {
     ZSTD_CCtx_params params = cctx->requestedParams;
-    ZSTD_prefixDict const prefixDict = cctx->prefixDict;
+    ZSTD_PrefixDict const prefixDict = cctx->prefixDict;
     FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */
     ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));   /* single usage */
     assert(prefixDict.dict==NULL || cctx->cdict==NULL);    /* only one can be set */
@@ -6407,7 +6409,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
         DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers);
         FORWARD_IF_ERROR( ZSTDMT_initCStream_internal(
                     cctx->mtctx,
-                    prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
+                    prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, prefixDict.loadMethod,
                     cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , "");
         cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0;
         cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize;

diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index ca5e2a4c5bf6fef724ee89d01204952ff8c37839..f71c47e880c3bf5c6a6aed212499424f8c32db76 100644 (file)
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -12,8 +12,8 @@
  * that shall **only** be used by modules within lib/compress.
  */
 
-#ifndef ZSTD_COMPRESS_H
-#define ZSTD_COMPRESS_H
+#ifndef ZSTD_COMPRESS_INTERNAL_H
+#define ZSTD_COMPRESS_INTERNAL_H
 
 /*-*************************************
 *  Dependencies
@@ -21,10 +21,10 @@
 #include "../common/zstd_internal.h"
 #include "zstd_cwksp.h"
 #ifdef ZSTD_MULTITHREAD
-#  include "zstdmt_compress.h"
+#  include "zstdmt_compress.h" /* ZSTDMT_CCtx */
 #endif
 #include "../common/bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */
-#include "zstd_preSplit.h" /* ZSTD_SLIPBLOCK_WORKSPACESIZE */
+#include "zstd_preSplit.h"  /* ZSTD_SLIPBLOCK_WORKSPACESIZE */
 
 /*-*************************************
 *  Constants
@@ -46,11 +46,12 @@
 typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
 typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
 
-typedef struct ZSTD_prefixDict_s {
+typedef struct {
     const void* dict;
     size_t dictSize;
     ZSTD_dictContentType_e dictContentType;
-} ZSTD_prefixDict;
+    ZSTD_dictLoadMethod_e loadMethod;
+} ZSTD_PrefixDict;
 
 typedef struct {
     void* dictBuffer;
@@ -467,7 +468,7 @@ typedef struct {
 
     U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS];
     ZSTD_entropyCTablesMetadata_t entropyMetadata;
-} ZSTD_blockSplitCtx;
+} ZSTD_BlockSplitCtx;
 
 struct ZSTD_CCtx_s {
     ZSTD_compressionStage_e stage;
@@ -525,7 +526,7 @@ struct ZSTD_CCtx_s {
     /* Dictionary */
     ZSTD_localDict localDict;
     const ZSTD_CDict* cdict;
-    ZSTD_prefixDict prefixDict;   /* single-usage dictionary */
+    ZSTD_PrefixDict prefixDict;   /* single-usage dictionary */
 
     /* Multi-threading */
 #ifdef ZSTD_MULTITHREAD
@@ -538,7 +539,7 @@ struct ZSTD_CCtx_s {
 #endif
 
     /* Workspace for block splitter */
-    ZSTD_blockSplitCtx blockSplitCtx;
+    ZSTD_BlockSplitCtx blockSplitCtx;
 
     /* Buffer for output from external sequence producer */
     ZSTD_Sequence* extSeqBuf;
@@ -553,7 +554,7 @@ typedef enum {
     ZSTD_extDict = 1,
     ZSTD_dictMatchState = 2,
     ZSTD_dedicatedDictSearch = 3
-} ZSTD_dictMode_e;
+} ZSTD_DictMode_e;
 
 typedef enum {
     ZSTD_cpm_noAttachDict = 0,  /* Compression with ZSTD_noDict or ZSTD_extDict.
@@ -578,7 +579,7 @@ typedef enum {
 typedef size_t (*ZSTD_BlockCompressor_f) (
         ZSTD_MatchState_t* bs, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize);
-ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
+ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e rowMatchfinderMode, ZSTD_DictMode_e dictMode);
 
 
 MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
@@ -1068,7 +1069,7 @@ MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
  * Inspects the provided matchState and figures out what dictMode should be
  * passed to the compressor.
  */
-MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms)
+MEM_STATIC ZSTD_DictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms)
 {
     return ZSTD_window_hasExtDict(ms->window) ?
         ZSTD_extDict :
@@ -1633,4 +1634,4 @@ size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx,
 size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
 
-#endif /* ZSTD_COMPRESS_H */
+#endif /* ZSTD_COMPRESS_INTERNAL_H */

diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 272ebe0ece7d9ceecaaa22a7ce0f067aa483c99b..1d5b37c20cc4446690c21d94facd50eeb1d4aab5 100644 (file)
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -74,7 +74,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
 void ZSTD_insertDUBT1(const ZSTD_MatchState_t* ms,
                  U32 curr, const BYTE* inputEnd,
                  U32 nbCompares, U32 btLow,
-                 const ZSTD_dictMode_e dictMode)
+                 const ZSTD_DictMode_e dictMode)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const bt = ms->chainTable;
@@ -168,7 +168,7 @@ size_t ZSTD_DUBT_findBetterDictMatch (
         size_t bestLength,
         U32 nbCompares,
         U32 const mls,
-        const ZSTD_dictMode_e dictMode)
+        const ZSTD_DictMode_e dictMode)
 {
     const ZSTD_MatchState_t * const dms = ms->dictMatchState;
     const ZSTD_compressionParameters* const dmsCParams = &dms->cParams;
@@ -244,7 +244,7 @@ size_t ZSTD_DUBT_findBestMatch(ZSTD_MatchState_t* ms,
                         const BYTE* const ip, const BYTE* const iend,
                         size_t* offBasePtr,
                         U32 const mls,
-                        const ZSTD_dictMode_e dictMode)
+                        const ZSTD_DictMode_e dictMode)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32*   const hashTable = ms->hashTable;
@@ -396,7 +396,7 @@ size_t ZSTD_BtFindBestMatch( ZSTD_MatchState_t* ms,
                 const BYTE* const ip, const BYTE* const iLimit,
                       size_t* offBasePtr,
                 const U32 mls /* template */,
-                const ZSTD_dictMode_e dictMode)
+                const ZSTD_DictMode_e dictMode)
 {
     DEBUGLOG(7, "ZSTD_BtFindBestMatch");
     if (ip < ms->window.base + ms->nextToUpdate) return 0;   /* skipped area */
@@ -668,7 +668,7 @@ size_t ZSTD_HcFindBestMatch(
                         ZSTD_MatchState_t* ms,
                         const BYTE* const ip, const BYTE* const iLimit,
                         size_t* offsetPtr,
-                        const U32 mls, const ZSTD_dictMode_e dictMode)
+                        const U32 mls, const ZSTD_DictMode_e dictMode)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const chainTable = ms->chainTable;
@@ -1142,7 +1142,7 @@ size_t ZSTD_RowFindBestMatch(
                         ZSTD_MatchState_t* ms,
                         const BYTE* const ip, const BYTE* const iLimit,
                         size_t* offsetPtr,
-                        const U32 mls, const ZSTD_dictMode_e dictMode,
+                        const U32 mls, const ZSTD_DictMode_e dictMode,
                         const U32 rowLog)
 {
     U32* const hashTable = ms->hashTable;
@@ -1492,7 +1492,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
     U32 const mls,
     U32 const rowLog,
     searchMethod_e const searchMethod,
-    ZSTD_dictMode_e const dictMode)
+    ZSTD_DictMode_e const dictMode)
 {
     if (dictMode == ZSTD_noDict) {
         ZSTD_SWITCH_SEARCH_METHOD(noDict)
@@ -1518,7 +1518,7 @@ size_t ZSTD_compressBlock_lazy_generic(
                         U32 rep[ZSTD_REP_NUM],
                         const void* src, size_t srcSize,
                         const searchMethod_e searchMethod, const U32 depth,
-                        ZSTD_dictMode_e const dictMode)
+                        ZSTD_DictMode_e const dictMode)
 {
     const BYTE* const istart = (const BYTE*)src;
     const BYTE* ip = istart;

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 3d7171b755bde44cf324fb31d4a791e8554f268a..09de5a9d9be44b854536f3cdbcb957c56e7f5c40 100644 (file)
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -562,7 +562,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
 void ZSTD_updateTree_internal(
                 ZSTD_MatchState_t* ms,
                 const BYTE* const ip, const BYTE* const iend,
-                const U32 mls, const ZSTD_dictMode_e dictMode)
+                const U32 mls, const ZSTD_DictMode_e dictMode)
 {
     const BYTE* const base = ms->window.base;
     U32 const target = (U32)(ip - base);
@@ -592,7 +592,7 @@ ZSTD_insertBtAndGetAllMatches (
                 ZSTD_MatchState_t* ms,
                 U32* nextToUpdate3,
                 const BYTE* const ip, const BYTE* const iLimit,
-                const ZSTD_dictMode_e dictMode,
+                const ZSTD_DictMode_e dictMode,
                 const U32 rep[ZSTD_REP_NUM],
                 const U32 ll0,  /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
                 const U32 lengthToBeat,
@@ -838,7 +838,7 @@ U32 ZSTD_btGetAllMatches_internal(
         const U32 rep[ZSTD_REP_NUM],
         U32 const ll0,
         U32 const lengthToBeat,
-        const ZSTD_dictMode_e dictMode,
+        const ZSTD_DictMode_e dictMode,
         const U32 mls)
 {
     assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls);
@@ -886,7 +886,7 @@ GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState)
     }
 
 static ZSTD_getAllMatchesFn
-ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const dictMode)
+ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_DictMode_e const dictMode)
 {
     ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = {
         ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict),
@@ -1079,7 +1079,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
                                U32 rep[ZSTD_REP_NUM],
                          const void* src, size_t srcSize,
                          const int optLevel,
-                         const ZSTD_dictMode_e dictMode)
+                         const ZSTD_DictMode_e dictMode)
 {
     optState_t* const optStatePtr = &ms->opt;
     const BYTE* const istart = (const BYTE*)src;
@@ -1445,7 +1445,7 @@ _shortestPath:   /* cur, last_pos, best_mlen, best_off have to be set */
 #ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
 static size_t ZSTD_compressBlock_opt0(
         ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
+        const void* src, size_t srcSize, const ZSTD_DictMode_e dictMode)
 {
     return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode);
 }
@@ -1454,7 +1454,7 @@ static size_t ZSTD_compressBlock_opt0(
 #ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
 static size_t ZSTD_compressBlock_opt2(
         ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
+        const void* src, size_t srcSize, const ZSTD_DictMode_e dictMode)
 {
     return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode);
 }

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index dbaffe5364392ac0f9363f1edb0ff53d4a5629be..4de6375131b1489e1d837eeec7b6db3143e0b208 100644 (file)
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1232,7 +1232,7 @@ static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params)
 
 size_t ZSTDMT_initCStream_internal(
         ZSTDMT_CCtx* mtctx,
-        const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
+        const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictLoadMethod_e dictLoadMethod,
         const ZSTD_CDict* cdict, ZSTD_CCtx_params params,
         unsigned long long pledgedSrcSize)
 {
@@ -1261,7 +1261,7 @@ size_t ZSTDMT_initCStream_internal(
     ZSTD_freeCDict(mtctx->cdictLocal);
     if (dict) {
         mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
-                                                    ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */
+                                                    dictLoadMethod, dictContentType, /* note : a loadPrefix becomes an internal CDict */
                                                     params.cParams, mtctx->cMem);
         mtctx->cdict = mtctx->cdictLocal;
         if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);

diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 91b489b9cb4f12df6a60728d22190a6e10751cc2..2158d614f30efcd9f2d931a92cfc0d7c25907b52 100644 (file)
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -64,7 +64,7 @@ size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
  *  even if they are not needed for the current compression.
  *  @return : 0, or an error code */
 size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx,
-                    const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
+                    const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictLoadMethod_e dictLoadMethod,
                     const ZSTD_CDict* cdict,
                     ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);