From: Yann Collet Date: Sat, 14 Oct 2017 02:01:58 +0000 (-0700) Subject: fixed ZSTD_initCStream_advanced() behavior, which depends on contentSizeFlag, X-Git-Tag: v1.3.3^2~52^2~8 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=213ef3b510b909b7dd95dc472e680ebea25bc450;p=thirdparty%2Fzstd.git fixed ZSTD_initCStream_advanced() behavior, which depends on contentSizeFlag, and a stream fuzzer test, which was incorrect (relied on 0 being unconditionnally transformed into `ZSTD_CONTENTSIZE_UNKNOWN`) --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 49b532b94..4ceba415c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -822,9 +822,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { - DEBUGLOG(4, "ZSTD_resetCCtx_internal"); + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u", (U32)pledgedSrcSize); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - DEBUGLOG(4, "pledgedSrcSize: %u", (U32)pledgedSrcSize); if (crp == ZSTDcrp_continue) { if (ZSTD_equivalentParams(params, zc->appliedParams)) { @@ -2500,12 +2499,15 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, return 0; /* ready to go */ } +/* ZSTD_resetCStream(): + * pledgedSrcSize == 0 means "unknown" */ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params params = zcs->requestedParams; - params.fParams.contentSizeFlag = (pledgedSrcSize > 0); + DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (U32)pledgedSrcSize); + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + params.fParams.contentSizeFlag = 1; params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0); - DEBUGLOG(4, "ZSTD_resetCStream"); return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); } @@ -2572,13 +2574,19 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0); /* note : will check that cdict != NULL */ } +/* ZSTD_initCStream_advanced() : + * pledgedSrcSize is optional: it can be 0, which means "unknown" if contentSizeFlag is set, or "unknown" if contentSizeFlag is not set. */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u", + (U32)pledgedSrcSize, params.fParams.contentSizeFlag); CHECK_F( ZSTD_checkCParams(params.cParams) ); + if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) + pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize); } diff --git a/lib/zstd.h b/lib/zstd.h index 7fafdbfc3..2866866be 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -734,7 +734,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspa ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/ ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */ + ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional: it can be 0, which means "unknown" if contentSizeFlag is set, or "unknown" if contentSizeFlag is not set. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 36eede293..3aea22709 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -559,7 +559,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++); { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); params.fParams.contentSizeFlag = 1; - CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0) ); + CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) ); } /* cstream advanced shall write content size = 0 */ inBuff.src = CNBuffer; inBuff.size = 0; @@ -833,10 +833,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); dict = srcBuffer + dictStart; } - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; + params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); } }