From: Yann Collet Date: Fri, 2 Feb 2018 22:24:56 +0000 (-0800) Subject: fileio: create dedicated function to generate zstd frames X-Git-Tag: v1.3.4~1^2~60^2~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=90eca318a776a1e365955e8a2c4f3245e5840bd7;p=thirdparty%2Fzstd.git fileio: create dedicated function to generate zstd frames like other formats --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e66b4df8a..67cc49b7b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -160,13 +160,6 @@ static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* CCtxParams, U64 sr CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM; } -static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) -{ - DEBUGLOG(4, "ZSTD_cLevelToCParams: level=%i", cctx->requestedParams.compressionLevel); - ZSTD_cLevelToCCtxParams_srcSize( - &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); -} - static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams) { DEBUGLOG(4, "ZSTD_cLevelToCCtxParams"); @@ -246,10 +239,51 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( return ERROR(parameter_outOfBound); \ } } + +static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) +{ + switch(param) + { + case ZSTD_p_compressionLevel: + case ZSTD_p_hashLog: + case ZSTD_p_chainLog: + case ZSTD_p_searchLog: + case ZSTD_p_minMatch: + case ZSTD_p_targetLength: + case ZSTD_p_compressionStrategy: + return 1; + + case ZSTD_p_format : + case ZSTD_p_windowLog: + case ZSTD_p_contentSizeFlag: + case ZSTD_p_checksumFlag: + case ZSTD_p_dictIDFlag: + case ZSTD_p_forceMaxWindow : + case ZSTD_p_nbWorkers: + case ZSTD_p_jobSize: + case ZSTD_p_overlapSizeLog: + case ZSTD_p_enableLongDistanceMatching: + case ZSTD_p_ldmHashLog: + case ZSTD_p_ldmMinMatch: + case ZSTD_p_ldmBucketSizeLog: + case ZSTD_p_ldmHashEveryLog: + return 0; + + default: + assert(0); + return 0; + } +} + size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value) { DEBUGLOG(4, "ZSTD_CCtx_setParameter (%u, %u)", (U32)param, value); - if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + if (cctx->streamStage != zcss_init) { + if (ZSTD_isUpdateAuthorized(param)) { + cctx->cParamsChanged = 1; + } else { + return ERROR(stage_wrong); + } } switch(param) { @@ -268,7 +302,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v case ZSTD_p_targetLength: case ZSTD_p_compressionStrategy: if (cctx->cdict) return ERROR(stage_wrong); - if (value>0) ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */ + if (value>0) { + ZSTD_cLevelToCCtxParams_srcSize(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); /* Optimize cParams when srcSize is known */ + } return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_contentSizeFlag: @@ -293,7 +329,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v case ZSTD_p_enableLongDistanceMatching: if (cctx->cdict) return ERROR(stage_wrong); - if (value>0) ZSTD_cLevelToCParams(cctx); + if (value>0) + ZSTD_cLevelToCCtxParams_srcSize(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); /* Optimize cParams when srcSize is known */ return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_ldmHashLog: @@ -477,11 +514,6 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( if (cctx->cdict) return ERROR(stage_wrong); cctx->requestedParams = *params; -#ifdef ZSTD_MULTITHREAD - if (cctx->mtctx) - ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing(cctx->mtctx, params); -#endif - return 0; } @@ -2497,7 +2529,8 @@ size_t ZSTD_compress_advanced_internal( const void* dict,size_t dictSize, ZSTD_CCtx_params params) { - DEBUGLOG(4, "ZSTD_compress_advanced_internal"); + DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", + (U32)srcSize); CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, params, srcSize, ZSTDb_not_buffered) ); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index eb651fb5e..ad2a4be0d 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -158,14 +158,14 @@ struct ZSTD_CCtx_params_s { /* For use with createCCtxParams() and freeCCtxParams() only */ ZSTD_customMem customMem; - }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; - U32 dictID; + int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ ZSTD_CCtx_params requestedParams; ZSTD_CCtx_params appliedParams; + U32 dictID; void* workSpace; size_t workSpaceSize; size_t blockSize; diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 90084b9b3..3542623e9 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -666,25 +666,16 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) return jobParams; } -/*! ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing() : - * Apply a ZSTD_CCtx_params to the compression context. - * This entry point is accessed while compression is ongoing, - * new parameters will be applied to next compression job. - * However, following parameters are NOT updated : - * - window size - * - pledgedSrcSize - * - nb threads - * - job size - * - overlap size - */ -void ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* params) +/*! ZSTDMT_MTCtx_updateParametersWhileCompressing() : + * Update compression level and parameters (except wlog) + * while compression is ongoing. + * New parameters will be applied to next compression job. */ +void ZSTDMT_MTCtx_updateParametersWhileCompressing(ZSTDMT_CCtx* mtctx, int compressionLevel, ZSTD_compressionParameters cParams) { - U32 const wlog = mtctx->params.cParams.windowLog; - U32 const nbWorkers = mtctx->params.nbWorkers; - mtctx->params = *params; + U32 const wlog = cParams.windowLog; + mtctx->params.cParams = cParams; mtctx->params.cParams.windowLog = wlog; /* Do not modify windowLog ! Frame must keep same wlog during the whole process ! */ - mtctx->params.nbWorkers = nbWorkers; /* Do not modify nbWorkers, it must remain synchronized with CCtx Pool ! */ - /* note : other parameters not updated are simply not used beyond initialization */ + mtctx->params.compressionLevel = compressionLevel; } /* ZSTDMT_getNbWorkers(): @@ -915,7 +906,7 @@ size_t ZSTDMT_initCStream_internal( const ZSTD_CDict* cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { - DEBUGLOG(2, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)", + DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)", (U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx); /* params are supposed to be fully validated at this point */ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 3fef2ed6a..c25521d38 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -121,17 +121,11 @@ size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_param * Also reset jobSize and overlapLog */ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers); -/*! ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing() : - * Apply a ZSTD_CCtx_params to the compression context. - * This works even during compression, and will be applied to next compression job. - * However, the following parameters will NOT be updated after compression has been started : - * - window size - * - pledgedSrcSize - * - nb threads - * - job size - * - overlap size - */ -void ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* params); +/*! ZSTDMT_MTCtx_updateParametersWhileCompressing() : + * Update compression level and parameters (except wlog) + * while compression is ongoing. + * New parameters will be applied to next compression job. */ +void ZSTDMT_MTCtx_updateParametersWhileCompressing(ZSTDMT_CCtx* mtctx, int compressionLevel, ZSTD_compressionParameters cParams); /* ZSTDMT_getNbWorkers(): * @return nb threads currently active in mtctx. diff --git a/lib/zstd.h b/lib/zstd.h index be93a7eeb..986b81980 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1049,8 +1049,10 @@ typedef enum { /*! ZSTD_CCtx_setParameter() : * Set one compression parameter, selected by enum ZSTD_cParameter. + * Setting a parameter is generally only possible during frame initialization (before starting compression), + * except for a few exceptions which can be updated during compression: compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. * Note : when `value` is an enum, cast it to unsigned for proper type checking. - * @result : informational value (typically, the one being set, possibly corrected), + * @result : informational value (typically, value being set clamped correctly), * or an error code (which can be tested with ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value); diff --git a/programs/fileio.c b/programs/fileio.c index 9fc0fc998..f1c9b9224 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -736,56 +736,22 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress, * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName */ -static int FIO_compressFilename_internal(cRess_t ress, - const char* dstFileName, const char* srcFileName, int compressionLevel) +static unsigned long long +FIO_compressZstdFrame(const cRess_t* ressPtr, + const char* srcFileName, U64 fileSize, + int compressionLevel, U64* readsize) { + cRess_t const ress = *ressPtr; FILE* const srcFile = ress.srcFile; FILE* const dstFile = ress.dstFile; - U64 readsize = 0; U64 compressedfilesize = 0; - U64 const fileSize = UTIL_getFileSize(srcFileName); ZSTD_EndDirective directive = ZSTD_e_continue; - DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize); - - switch (g_compressionType) { - case FIO_zstdCompression: - break; - - case FIO_gzipCompression: -#ifdef ZSTD_GZCOMPRESS - compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); -#else - (void)compressionLevel; - EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", - srcFileName); -#endif - goto finish; - - case FIO_xzCompression: - case FIO_lzmaCompression: -#ifdef ZSTD_LZMACOMPRESS - compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression); -#else - (void)compressionLevel; - EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", - srcFileName); -#endif - goto finish; - - case FIO_lz4Compression: -#ifdef ZSTD_LZ4COMPRESS - compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize); -#else - (void)compressionLevel; - EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", - srcFileName); -#endif - goto finish; - } + DISPLAYLEVEL(6, "compression using zstd format \n"); /* init */ if (fileSize != UTIL_FILESIZE_UNKNOWN) ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); + (void)compressionLevel; (void)srcFileName; /* Main compression loop */ do { @@ -794,9 +760,9 @@ static int FIO_compressFilename_internal(cRess_t ress, size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; DISPLAYLEVEL(6, "fread %u bytes from source \n", (U32)inSize); - readsize += inSize; + *readsize += inSize; - if (inSize == 0 || (fileSize != UTIL_FILESIZE_UNKNOWN && readsize == fileSize)) + if ((inSize == 0) || (*readsize == fileSize)) directive = ZSTD_e_end; result = 1; @@ -810,7 +776,7 @@ static int FIO_compressFilename_internal(cRess_t ress, if (outBuff.pos) { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); if (sizeCheck!=outBuff.pos) - EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); + EXM_THROW(25, "Write error : cannot write compressed block"); compressedfilesize += outBuff.pos; } if (READY_FOR_UPDATE()) { @@ -824,10 +790,67 @@ static int FIO_compressFilename_internal(cRess_t ress, } } while (directive != ZSTD_e_end); -finish: + return compressedfilesize; +} + +/*! FIO_compressFilename_internal() : + * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. + * @return : 0 : compression completed correctly, + * 1 : missing or pb opening srcFileName + */ +static int +FIO_compressFilename_internal(cRess_t ress, + const char* dstFileName, const char* srcFileName, + int compressionLevel) +{ + U64 readsize = 0; + U64 compressedfilesize = 0; + U64 const fileSize = UTIL_getFileSize(srcFileName); + DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize); + + /* compression format selection */ + switch (g_compressionType) { + default: + case FIO_zstdCompression: + compressedfilesize = FIO_compressZstdFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); + break; + + case FIO_gzipCompression: +#ifdef ZSTD_GZCOMPRESS + compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); +#else + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", + srcFileName); +#endif + break; + + case FIO_xzCompression: + case FIO_lzmaCompression: +#ifdef ZSTD_LZMACOMPRESS + compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression); +#else + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", + srcFileName); +#endif + break; + + case FIO_lz4Compression: +#ifdef ZSTD_LZ4COMPRESS + compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize); +#else + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", + srcFileName); +#endif + break; + } + /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", + srcFileName, (double)compressedfilesize / (readsize+(!readsize)/*avoid div by zero*/) * 100, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName);