From: Stella Lau Date: Wed, 30 Aug 2017 21:36:54 +0000 (-0700) Subject: Pass dictMode to ZSTDMT_initCStream; fix nits X-Git-Tag: fuzz-corpora2~29^2^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=90a31bfa162ce0aaac9f3645ab553bbc417b5bc3;p=thirdparty%2Fzstd.git Pass dictMode to ZSTDMT_initCStream; fix nits - Return error code in estimate{CCtx,CStream}Size functions --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 206e0976a..5adeb480b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -233,7 +233,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( ZSTD_compressionParameters cParams) { ZSTD_CCtx_params cctxParams; - memset(&cctxParams, 0, sizeof(ZSTD_CCtx_params)); + memset(&cctxParams, 0, sizeof(cctxParams)); cctxParams.cParams = cParams; cctxParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; return cctxParams; @@ -271,7 +271,7 @@ size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params) size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, int compressionLevel) { if (!cctxParams) { return ERROR(GENERIC); } - memset(cctxParams, 0, sizeof(ZSTD_CCtx_params)); + memset(cctxParams, 0, sizeof(*cctxParams)); cctxParams->compressionLevel = compressionLevel; return 0; } @@ -280,7 +280,7 @@ size_t ZSTD_initCCtxParams_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameter { if (!cctxParams) { return ERROR(GENERIC); } CHECK_F( ZSTD_checkCParams(params.cParams) ); - memset(cctxParams, 0, sizeof(ZSTD_CCtx_params)); + memset(cctxParams, 0, sizeof(*cctxParams)); cctxParams->cParams = params.cParams; cctxParams->fParams = params.fParams; cctxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM; @@ -543,7 +543,7 @@ size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); cctx->cdict = cdict; - memset(&cctx->prefixDict, 0, sizeof(ZSTD_prefixDict)); /* exclusive */ + memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* exclusive */ return 0; } @@ -657,9 +657,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params) { /* Estimate CCtx size is supported for single-threaded compression only. */ - if (params->nbThreads > 1) { - return 0; - } + if (params->nbThreads > 1) { return ERROR(GENERIC); } { ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(*params, 0, 0); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); @@ -701,9 +699,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel) size_t ZSTD_estimateCStreamSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params) { - if (params->nbThreads > 1) { - return 0; - } + if (params->nbThreads > 1) { return ERROR(GENERIC); } { size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_usingCCtxParams(params); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog); size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize; @@ -3171,7 +3167,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) { - ZSTD_compressionParameters cParams = + ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(cctx->appliedParams, 0, 0); return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog); } @@ -3413,7 +3409,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) { - ZSTD_CCtx_params cctxParams = + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dm_auto, cctxParams, @@ -3423,7 +3419,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - ZSTD_CCtx_params cctxParams = + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, cctxParams, 0, ZSTDb_not_buffered); @@ -3505,7 +3501,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params) { - ZSTD_CCtx_params cctxParams = + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, @@ -3611,10 +3607,7 @@ static size_t ZSTD_initCDict_internal( } cdict->dictContentSize = dictSize; - { ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, - 0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */ - ZSTD_CCtx_params cctxParams = cdict->refContext->requestedParams; - cctxParams.fParams = fParams; + { ZSTD_CCtx_params cctxParams = cdict->refContext->requestedParams; cctxParams.cParams = cParams; CHECK_F( ZSTD_compressBegin_internal(cdict->refContext, cdict->dictContent, dictSize, dictMode, @@ -3923,7 +3916,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) { - ZSTD_CCtx_params cctxParams = + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); CHECK_F( ZSTD_checkCParams(params.cParams) ); return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize); @@ -3932,7 +3925,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - ZSTD_CCtx_params cctxParams = + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0); } @@ -3940,9 +3933,9 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params cctxParams; - ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); - params.fParams.contentSizeFlag = (pledgedSrcSize>0); + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + cctxParams.fParams.contentSizeFlag = (pledgedSrcSize>0); return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, cctxParams, pledgedSrcSize); } @@ -4122,7 +4115,8 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf * must receive dict, or cdict, or none, but not both. * @return : 0, or an error code */ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, - const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, + const ZSTD_CDict* cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); @@ -4142,7 +4136,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ZSTD_getCParamsFromCCtxParams( cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); - memset(&cctx->prefixDict, 0, sizeof(ZSTD_prefixDict)); /* single usage */ + memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ #ifdef ZSTD_MULTITHREAD @@ -4155,7 +4149,8 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads); CHECK_F( ZSTDMT_initCStream_internal( - cctx->mtctx, prefixDict.dict, prefixDict.dictSize, + cctx->mtctx, + prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); cctx->streamStage = zcss_load; cctx->appliedParams.nbThreads = params.nbThreads; diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 86afe5065..166f99d72 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -191,7 +191,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) static ZSTD_CCtx_params ZSTDMT_makeJobCCtxParams(ZSTD_CCtx_params const params) { ZSTD_CCtx_params jobParams; - memset(&jobParams, 0, sizeof(ZSTD_CCtx_params)); + memset(&jobParams, 0, sizeof(jobParams)); jobParams.cParams = params.cParams; jobParams.fParams = params.fParams; @@ -737,7 +737,8 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) } size_t ZSTDMT_initCStream_internal( - ZSTDMT_CCtx* zcs, const void* dict, size_t dictSize, + ZSTDMT_CCtx* zcs, + const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { @@ -768,7 +769,7 @@ size_t ZSTDMT_initCStream_internal( DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal); zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dm_rawContent, /* note : a loadPrefix becomes an internal CDict */ + ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */ params.cParams, zcs->cMem); zcs->cdict = zcs->cdictLocal; if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); @@ -807,7 +808,7 @@ size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, DEBUGLOG(5, "ZSTDMT_initCStream_advanced"); cctxParams.cParams = params.cParams; cctxParams.fParams = params.fParams; - return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, NULL, + return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, ZSTD_dm_auto, NULL, cctxParams, pledgedSrcSize); } @@ -820,7 +821,7 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, cctxParams.cParams = ZSTD_getCParamsFromCDict(cdict); cctxParams.fParams = fParams; if (cdict==NULL) return ERROR(dictionary_wrong); /* method incompatible with NULL cdict */ - return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, cdict, + return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, ZSTD_dm_auto, cdict, cctxParams, pledgedSrcSize); } @@ -831,7 +832,7 @@ size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize) { if (zcs->params.nbThreads==1) return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize); - return ZSTDMT_initCStream_internal(zcs, NULL, 0, 0, zcs->params, + return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, 0, zcs->params, pledgedSrcSize); } @@ -840,7 +841,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) { ZSTD_CCtx_params cctxParams = zcs->params; cctxParams.cParams = params.cParams; cctxParams.fParams = params.fParams; - return ZSTDMT_initCStream_internal(zcs, NULL, 0, NULL, cctxParams, 0); + return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, 0); } diff --git a/lib/zstd.h b/lib/zstd.h index e062f6d2b..c11964408 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -498,7 +498,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced_usingCParams() can provide a tighter estimation. * ZSTD_estimateCCtxSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCCtxSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return 0 if ZSTD_p_nbThreads is set to a value > 1. + * ZSTD_estimateCCtxSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1. * Note : CCtx estimation is only correct for single-threaded compression */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced_usingCParams(ZSTD_compressionParameters cParams); @@ -510,7 +510,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced_usingCParams() can provide a tighter estimation. * ZSTD_estimateCStreamSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCStreamSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return 0 if ZSTD_p_nbThreads is set to a value > 1. + * ZSTD_estimateCStreamSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1. * Note : CStream estimation is only correct for single-threaded compression. * ZSTD_DStream memory budget depends on window Size. * This information can be passed manually, using ZSTD_estimateDStreamSize, @@ -727,9 +727,9 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ -ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */ +ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/ ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ + ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index f8683ac78..8be6a5910 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -149,6 +149,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo U32 testNb = 1; ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); + ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2); + ZSTD_inBuffer inBuff, inBuff2; ZSTD_outBuffer outBuff; buffer_t dictionary = g_nullBuffer; @@ -605,6 +607,25 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; DISPLAYLEVEL(3, "OK \n"); + /* Basic multithreading compression test */ + DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + { ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); + size_t const r = ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize); + if (ZSTD_isError(r)) goto _output_error; } + outBuff.dst = (char*)(compressedBuffer); + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const r = ZSTDMT_compressStream_generic(mtctx, &outBuff, &inBuff, ZSTD_e_end); + if (ZSTD_isError(r)) goto _output_error; } + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTDMT_endStream(mtctx, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + DISPLAYLEVEL(3, "OK \n"); + + /* Overlen overwriting window data bug */ DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++); { /* This test has a window size of 1024 bytes and consists of 3 blocks: @@ -643,6 +664,7 @@ _end: FUZ_freeDictionary(dictionary); ZSTD_freeCStream(zc); ZSTD_freeDStream(zd); + ZSTDMT_freeCCtx(mtctx); free(CNBuffer); free(compressedBuffer); free(decodedBuffer);