From 23e5f80390db9a3a65485933d255e163d2dab519 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 5 Feb 2025 18:47:26 -0800 Subject: [PATCH] Revert "pass dictionary loading method as parameter" This reverts commit 821fc567f93a415e9fbe856271ccd452ee7acf07. --- lib/compress/zstd_compress.c | 12 +++++------- lib/compress/zstd_compress_internal.h | 27 +++++++++++++-------------- lib/compress/zstd_lazy.c | 16 ++++++++-------- lib/compress/zstd_opt.c | 14 +++++++------- lib/compress/zstdmt_compress.c | 16 ++++++++++++++-- lib/compress/zstdmt_compress.h | 2 +- 6 files changed, 48 insertions(+), 39 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c560c902a..d928b1d3e 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1354,12 +1354,10 @@ size_t ZSTD_CCtx_refPrefix_advanced( RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, "Can't ref a prefix when ctx not in init stage."); ZSTD_clearAllDicts(cctx); - if (prefixSize > 0) { - RETURN_ERROR_IF(prefix == NULL, dictionary_wrong, "Invalid prefix pointer"); + if (prefix != NULL && prefixSize > 0) { cctx->prefixDict.dict = prefix; cctx->prefixDict.dictSize = prefixSize; cctx->prefixDict.dictContentType = dictContentType; - cctx->prefixDict.loadMethod = ZSTD_dlm_byRef; } return 0; } @@ -3068,7 +3066,7 @@ ZSTD_entropyCompressSeqStore( /* ZSTD_selectBlockCompressor() : * Not static, but internal use only (used by long distance matcher) * assumption : strat is a valid strategy */ -ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e useRowMatchFinder, ZSTD_DictMode_e dictMode) +ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode) { static const ZSTD_BlockCompressor_f blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { { ZSTD_compressBlock_fast /* default for 0 */, @@ -3300,7 +3298,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) } /* select and store sequences */ - { ZSTD_DictMode_e const dictMode = ZSTD_matchState_dictMode(ms); + { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); size_t lastLLSize; { int i; for (i = 0; i < ZSTD_REP_NUM; ++i) @@ -6353,7 +6351,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, size_t inSize) { ZSTD_CCtx_params params = cctx->requestedParams; - ZSTD_PrefixDict const prefixDict = cctx->prefixDict; + ZSTD_prefixDict const prefixDict = cctx->prefixDict; FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ @@ -6409,7 +6407,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( cctx->mtctx, - prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, prefixDict.loadMethod, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0; cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index f71c47e88..ca5e2a4c5 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -12,8 +12,8 @@ * that shall **only** be used by modules within lib/compress. */ -#ifndef ZSTD_COMPRESS_INTERNAL_H -#define ZSTD_COMPRESS_INTERNAL_H +#ifndef ZSTD_COMPRESS_H +#define ZSTD_COMPRESS_H /*-************************************* * Dependencies @@ -21,10 +21,10 @@ #include "../common/zstd_internal.h" #include "zstd_cwksp.h" #ifdef ZSTD_MULTITHREAD -# include "zstdmt_compress.h" /* ZSTDMT_CCtx */ +# include "zstdmt_compress.h" #endif #include "../common/bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */ -#include "zstd_preSplit.h" /* ZSTD_SLIPBLOCK_WORKSPACESIZE */ +#include "zstd_preSplit.h" /* ZSTD_SLIPBLOCK_WORKSPACESIZE */ /*-************************************* * Constants @@ -46,12 +46,11 @@ typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; -typedef struct { +typedef struct ZSTD_prefixDict_s { const void* dict; size_t dictSize; ZSTD_dictContentType_e dictContentType; - ZSTD_dictLoadMethod_e loadMethod; -} ZSTD_PrefixDict; +} ZSTD_prefixDict; typedef struct { void* dictBuffer; @@ -468,7 +467,7 @@ typedef struct { U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS]; ZSTD_entropyCTablesMetadata_t entropyMetadata; -} ZSTD_BlockSplitCtx; +} ZSTD_blockSplitCtx; struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; @@ -526,7 +525,7 @@ struct ZSTD_CCtx_s { /* Dictionary */ ZSTD_localDict localDict; const ZSTD_CDict* cdict; - ZSTD_PrefixDict prefixDict; /* single-usage dictionary */ + ZSTD_prefixDict prefixDict; /* single-usage dictionary */ /* Multi-threading */ #ifdef ZSTD_MULTITHREAD @@ -539,7 +538,7 @@ struct ZSTD_CCtx_s { #endif /* Workspace for block splitter */ - ZSTD_BlockSplitCtx blockSplitCtx; + ZSTD_blockSplitCtx blockSplitCtx; /* Buffer for output from external sequence producer */ ZSTD_Sequence* extSeqBuf; @@ -554,7 +553,7 @@ typedef enum { ZSTD_extDict = 1, ZSTD_dictMatchState = 2, ZSTD_dedicatedDictSearch = 3 -} ZSTD_DictMode_e; +} ZSTD_dictMode_e; typedef enum { ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict. @@ -579,7 +578,7 @@ typedef enum { typedef size_t (*ZSTD_BlockCompressor_f) ( ZSTD_MatchState_t* bs, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); -ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e rowMatchfinderMode, ZSTD_DictMode_e dictMode); +ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode); MEM_STATIC U32 ZSTD_LLcode(U32 litLength) @@ -1069,7 +1068,7 @@ MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) * Inspects the provided matchState and figures out what dictMode should be * passed to the compressor. */ -MEM_STATIC ZSTD_DictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms) +MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms) { return ZSTD_window_hasExtDict(ms->window) ? ZSTD_extDict : @@ -1634,4 +1633,4 @@ size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -#endif /* ZSTD_COMPRESS_INTERNAL_H */ +#endif /* ZSTD_COMPRESS_H */ diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 1d5b37c20..272ebe0ec 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -74,7 +74,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_insertDUBT1(const ZSTD_MatchState_t* ms, U32 curr, const BYTE* inputEnd, U32 nbCompares, U32 btLow, - const ZSTD_DictMode_e dictMode) + const ZSTD_dictMode_e dictMode) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const bt = ms->chainTable; @@ -168,7 +168,7 @@ size_t ZSTD_DUBT_findBetterDictMatch ( size_t bestLength, U32 nbCompares, U32 const mls, - const ZSTD_DictMode_e dictMode) + const ZSTD_dictMode_e dictMode) { const ZSTD_MatchState_t * const dms = ms->dictMatchState; const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; @@ -244,7 +244,7 @@ size_t ZSTD_DUBT_findBestMatch(ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iend, size_t* offBasePtr, U32 const mls, - const ZSTD_DictMode_e dictMode) + const ZSTD_dictMode_e dictMode) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; @@ -396,7 +396,7 @@ size_t ZSTD_BtFindBestMatch( ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, size_t* offBasePtr, const U32 mls /* template */, - const ZSTD_DictMode_e dictMode) + const ZSTD_dictMode_e dictMode) { DEBUGLOG(7, "ZSTD_BtFindBestMatch"); if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ @@ -668,7 +668,7 @@ size_t ZSTD_HcFindBestMatch( ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, - const U32 mls, const ZSTD_DictMode_e dictMode) + const U32 mls, const ZSTD_dictMode_e dictMode) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const chainTable = ms->chainTable; @@ -1142,7 +1142,7 @@ size_t ZSTD_RowFindBestMatch( ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, - const U32 mls, const ZSTD_DictMode_e dictMode, + const U32 mls, const ZSTD_dictMode_e dictMode, const U32 rowLog) { U32* const hashTable = ms->hashTable; @@ -1492,7 +1492,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax( U32 const mls, U32 const rowLog, searchMethod_e const searchMethod, - ZSTD_DictMode_e const dictMode) + ZSTD_dictMode_e const dictMode) { if (dictMode == ZSTD_noDict) { ZSTD_SWITCH_SEARCH_METHOD(noDict) @@ -1518,7 +1518,7 @@ size_t ZSTD_compressBlock_lazy_generic( U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, const searchMethod_e searchMethod, const U32 depth, - ZSTD_DictMode_e const dictMode) + ZSTD_dictMode_e const dictMode) { const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 09de5a9d9..3d7171b75 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -562,7 +562,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_updateTree_internal( ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iend, - const U32 mls, const ZSTD_DictMode_e dictMode) + const U32 mls, const ZSTD_dictMode_e dictMode) { const BYTE* const base = ms->window.base; U32 const target = (U32)(ip - base); @@ -592,7 +592,7 @@ ZSTD_insertBtAndGetAllMatches ( ZSTD_MatchState_t* ms, U32* nextToUpdate3, const BYTE* const ip, const BYTE* const iLimit, - const ZSTD_DictMode_e dictMode, + const ZSTD_dictMode_e dictMode, const U32 rep[ZSTD_REP_NUM], const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ const U32 lengthToBeat, @@ -838,7 +838,7 @@ U32 ZSTD_btGetAllMatches_internal( const U32 rep[ZSTD_REP_NUM], U32 const ll0, U32 const lengthToBeat, - const ZSTD_DictMode_e dictMode, + const ZSTD_dictMode_e dictMode, const U32 mls) { assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls); @@ -886,7 +886,7 @@ GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState) } static ZSTD_getAllMatchesFn -ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_DictMode_e const dictMode) +ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const dictMode) { ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = { ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict), @@ -1079,7 +1079,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, const int optLevel, - const ZSTD_DictMode_e dictMode) + const ZSTD_dictMode_e dictMode) { optState_t* const optStatePtr = &ms->opt; const BYTE* const istart = (const BYTE*)src; @@ -1445,7 +1445,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ #ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR static size_t ZSTD_compressBlock_opt0( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize, const ZSTD_DictMode_e dictMode) + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) { return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode); } @@ -1454,7 +1454,7 @@ static size_t ZSTD_compressBlock_opt0( #ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR static size_t ZSTD_compressBlock_opt2( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize, const ZSTD_DictMode_e dictMode) + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) { return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode); } diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 8725b9071..0f1fe6d74 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -1248,7 +1248,7 @@ static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) size_t ZSTDMT_initCStream_internal( ZSTDMT_CCtx* mtctx, - const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictLoadMethod_e dictLoadMethod, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { @@ -1274,6 +1274,18 @@ size_t ZSTDMT_initCStream_internal( mtctx->params = params; mtctx->frameContentSize = pledgedSrcSize; + ZSTD_freeCDict(mtctx->cdictLocal); + if (dict) { + mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */ + params.cParams, mtctx->cMem); + mtctx->cdict = mtctx->cdictLocal; + if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + mtctx->cdictLocal = NULL; + mtctx->cdict = cdict; + } + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); mtctx->targetSectionSize = params.jobSize; @@ -1347,7 +1359,7 @@ size_t ZSTDMT_initCStream_internal( } else { /* note : a loadPrefix becomes an internal CDict */ mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, - dictLoadMethod, dictContentType, + ZSTD_dlm_byRef, dictContentType, params.cParams, mtctx->cMem); mtctx->cdict = mtctx->cdictLocal; if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 2158d614f..91b489b9c 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -64,7 +64,7 @@ size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); * even if they are not needed for the current compression. * @return : 0, or an error code */ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx, - const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictLoadMethod_e dictLoadMethod, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); -- 2.47.2