From: Stella Lau Date: Fri, 25 Aug 2017 17:48:07 +0000 (-0700) Subject: Remove ZSTD_p_refDictContent and dictContentByRef X-Git-Tag: fuzz-corpora2~29^2^2~16 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=eb7bbab36a1a5b81be860234cd6ec602184d4fc7;p=thirdparty%2Fzstd.git Remove ZSTD_p_refDictContent and dictContentByRef --- diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 4e2403bee..393a17247 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -294,7 +294,6 @@ struct ZSTD_CCtx_params_s { U32 forceWindow; /* force back-references to respect limit of * 1<requestedParams, param, value); case ZSTD_p_dictMode: - case ZSTD_p_refDictContent: if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */ return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); @@ -463,11 +462,6 @@ size_t ZSTD_CCtxParam_setParameter( params->dictMode = (ZSTD_dictMode_e)value; return 0; - case ZSTD_p_refDictContent : - /* dictionary content will be referenced, instead of copied */ - params->dictContentByRef = value > 0; - return 0; - case ZSTD_p_forceMaxWindow : params->forceWindow = value > 0; return 0; @@ -514,7 +508,6 @@ size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params /* Assume dictionary parameters are validated */ cctx->requestedParams.dictMode = params->dictMode; - cctx->requestedParams.dictContentByRef = params->dictContentByRef; /* Set force window explicitly since it sets cctx->loadedDictEnd */ CHECK_F( ZSTD_CCtx_setParameter( @@ -541,7 +534,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo return 0; } -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +size_t ZSTD_CCtx_loadDictionary_internal( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize, unsigned byReference) { if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */ @@ -557,7 +551,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s ZSTD_getCParams(cctx->requestedParams.compressionLevel, 0, dictSize); cctx->cdictLocal = ZSTD_createCDict_advanced( dict, dictSize, - cctx->requestedParams.dictContentByRef, + byReference, cctx->requestedParams.dictMode, cParams, cctx->customMem); cctx->cdict = cctx->cdictLocal; @@ -567,6 +561,18 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s return 0; } +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_internal(cctx, dict, dictSize, 1); +} + +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_internal(cctx, dict, dictSize, 0); +} + + size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); @@ -3896,7 +3902,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, ZSTD_freeCDict(zcs->cdictLocal); zcs->cdictLocal = ZSTD_createCDict_advanced( dict, dictSize, - params.dictContentByRef, params.dictMode, + 0 /* byReference */, params.dictMode, params.cParams, zcs->customMem); zcs->cdict = zcs->cdictLocal; if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index f7f02cfbb..fae17e656 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -344,7 +344,7 @@ void ZSTDMT_compressChunk(void* jobDescription) { ZSTD_CCtx_params jobParams = job->params; /* Force loading dictionary in "content-only" mode (no header analysis) */ size_t const dictModeError = - ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_dictMode, 1); + ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_dictMode, (U32)ZSTD_dm_rawContent); size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk); /* Note: ZSTD_setCCtxParameter() should not be used here. diff --git a/lib/zstd.h b/lib/zstd.h index 03994f9a8..01fbb18cd 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -970,8 +970,6 @@ typedef enum { /* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */ ZSTD_p_dictMode=300, /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e. * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */ - ZSTD_p_refDictContent, /* Dictionary content will be referenced, instead of copied (default:0==byCopy). - * It requires that dictionary buffer outlives its users */ /* multi-threading parameters */ ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1) @@ -1015,8 +1013,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, * meaning "return to no-dictionary mode". - * Note 1 : `dict` content will be copied internally, - * except if ZSTD_p_refDictContent is set before loading. + * Note 1 : `dict` content will be copied internally. Use + * ZSTD_CCtx_loadDictionary_byReference() to reference dictionary + * content instead. * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters. * For this reason, compression parameters cannot be changed anymore after loading a dictionary. * It's also a CPU-heavy operation, with non-negligible impact on latency. @@ -1024,6 +1023,12 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo * To return to "no-dictionary" situation, load a NULL dictionary */ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary() except dictionary content will be + * referenced, instead of copied. The dictionary buffer must outlive its users. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + /*! ZSTD_CCtx_refCDict() : * Reference a prepared dictionary, to be used for all next compression jobs. * Note that compression parameters are enforced from within CDict, diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 98e121949..0373676d0 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1366,7 +1366,6 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize); - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); /* multi-threading parameters */ { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax); @@ -1386,7 +1385,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double } if (FUZ_rand(&lseed) & 1) { - CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); + if (FUZ_rand(&lseed) & 1) { + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); + } else { + CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) ); + } if (dict && dictSize) { /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */ if (useOpaqueAPI) {