From: Nick Terrell Date: Wed, 13 Mar 2019 21:05:18 +0000 (-0700) Subject: Wrap the new advanced api completely X-Git-Tag: v1.4.0^2~25^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e55da9e9638ef504f93a0ed448a47d0a8d745657;p=thirdparty%2Fzstd.git Wrap the new advanced api completely --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 987c7b90a..62dab8003 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3769,13 +3769,17 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, /* ZSTD_resetCStream(): * pledgedSrcSize == 0 means "unknown" */ -size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) { - ZSTD_CCtx_params params = zcs->requestedParams; + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); - if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; - params.fParams.contentSizeFlag = 1; - return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + return 0; } /*! ZSTD_initCStream_internal() : @@ -3787,31 +3791,18 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); - params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + zcs->requestedParams = params; assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - if (dict && dictSize >= 8) { - DEBUGLOG(4, "loading dictionary of size %u", (unsigned)dictSize); - RETURN_ERROR_IF( - zcs->staticSize, memory_allocation, - "static CCtx: incompatible with internal cdict creation"); - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - params.cParams, zcs->customMem); - zcs->cdict = zcs->cdictLocal; - RETURN_ERROR_IF(zcs->cdictLocal == NULL, memory_allocation); + if (dict) { + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); } else { - if (cdict) { - params.cParams = ZSTD_getCParamsFromCDict(cdict); /* cParams are enforced from cdict; it includes windowLog */ - } - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = NULL; - zcs->cdict = cdict; + /* Dictionary is cleared if !cdict */ + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); } - - return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); + return 0; } /* ZSTD_initCStream_usingCDict_advanced() : @@ -3822,23 +3813,20 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); - RETURN_ERROR_IF(!cdict, dictionary_wrong, - "cannot handle NULL cdict (does not know what to do)"); - { ZSTD_CCtx_params params = zcs->requestedParams; - params.cParams = ZSTD_getCParamsFromCDict(cdict); - params.fParams = fParams; - return ZSTD_initCStream_internal(zcs, - NULL, 0, cdict, - params, pledgedSrcSize); - } + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + zcs->requestedParams.fParams = fParams; + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); + return 0; } /* note : cdict must outlive compression session */ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) { - ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksum */, 0 /* hideDictID */ }; DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); - return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); /* note : will check that cdict != NULL */ + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); + return 0; } @@ -3848,33 +3836,53 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize) + ZSTD_parameters params, unsigned long long pss) { - DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u", - (unsigned)pledgedSrcSize, params.fParams.contentSizeFlag); + /* for compatibility with older programs relying on this behavior. + * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. + * This line will be removed in the future. + */ + U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); - if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */ zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); + return 0; } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, zcs->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN); + DEBUGLOG(4, "ZSTD_initCStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); + return 0; } size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) { - U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */ - ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); - return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, zcs->requestedParams, pledgedSrcSize); + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_srcSize"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + return 0; } size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) { DEBUGLOG(4, "ZSTD_initCStream"); - return ZSTD_initCStream_srcSize(zcs, compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + return 0; } /*====== Compression ======*/ @@ -3898,10 +3906,10 @@ static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, * internal function for all *compressStream*() variants * non-static, because can be called from zstdmt_compress.c * @return : hint size for next input */ -size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective const flushMode) +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) { const char* const istart = (const char*)input->src; const char* const iend = istart + input->size; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index a828de3b4..c90034d4f 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -808,13 +808,6 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, void ZSTD_resetSeqStore(seqStore_t* ssPtr); -/*! ZSTD_compressStream_generic() : - * Private use only. To be called from zstdmt_compress.c in single-thread mode. */ -size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective const flushMode); - /*! ZSTD_getCParamsFromCDict() : * as the name implies */ ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index dab3a9508..7e2c78984 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -1967,7 +1967,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, assert(input->pos <= input->size); if (mtctx->singleBlockingThread) { /* delegate to single-thread (synchronous) */ - return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp); + return ZSTD_compressStream2(mtctx->cctxPool->cctx[0], output, input, endOp); } if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index b099e2870..e26375e07 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -495,7 +495,7 @@ static int basicUnitTests(U32 seed, double compressibility) /* _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - ZSTD_initCStream_srcSize(zc, 1, CNBufferSize); + CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) ); outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -503,11 +503,14 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff.size = CNBufferSize; inBuff.pos = 0; CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - { size_t const r = ZSTD_endStream(zc, &outBuff); - if (r != 0) goto _output_error; } /* error, or some data not flushed */ - { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos); - if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */ + CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed"); + { size_t const r = ZSTD_endStream(zc, &outBuff); + CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r); + } + { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos); + CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!"); + CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize); + } DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); /* wrong _srcSize compression test */