CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM;
}
-static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
-{
- DEBUGLOG(4, "ZSTD_cLevelToCParams: level=%i", cctx->requestedParams.compressionLevel);
- ZSTD_cLevelToCCtxParams_srcSize(
- &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1);
-}
-
static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams)
{
DEBUGLOG(4, "ZSTD_cLevelToCCtxParams");
return ERROR(parameter_outOfBound); \
} }
+
+static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
+{
+ switch(param)
+ {
+ case ZSTD_p_compressionLevel:
+ case ZSTD_p_hashLog:
+ case ZSTD_p_chainLog:
+ case ZSTD_p_searchLog:
+ case ZSTD_p_minMatch:
+ case ZSTD_p_targetLength:
+ case ZSTD_p_compressionStrategy:
+ return 1;
+
+ case ZSTD_p_format :
+ case ZSTD_p_windowLog:
+ case ZSTD_p_contentSizeFlag:
+ case ZSTD_p_checksumFlag:
+ case ZSTD_p_dictIDFlag:
+ case ZSTD_p_forceMaxWindow :
+ case ZSTD_p_nbWorkers:
+ case ZSTD_p_jobSize:
+ case ZSTD_p_overlapSizeLog:
+ case ZSTD_p_enableLongDistanceMatching:
+ case ZSTD_p_ldmHashLog:
+ case ZSTD_p_ldmMinMatch:
+ case ZSTD_p_ldmBucketSizeLog:
+ case ZSTD_p_ldmHashEveryLog:
+ return 0;
+
+ default:
+ assert(0);
+ return 0;
+ }
+}
+
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
{
DEBUGLOG(4, "ZSTD_CCtx_setParameter (%u, %u)", (U32)param, value);
- if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+ if (cctx->streamStage != zcss_init) {
+ if (ZSTD_isUpdateAuthorized(param)) {
+ cctx->cParamsChanged = 1;
+ } else {
+ return ERROR(stage_wrong);
+ } }
switch(param)
{
case ZSTD_p_targetLength:
case ZSTD_p_compressionStrategy:
if (cctx->cdict) return ERROR(stage_wrong);
- if (value>0) ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */
+ if (value>0) {
+ ZSTD_cLevelToCCtxParams_srcSize(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); /* Optimize cParams when srcSize is known */
+ }
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_contentSizeFlag:
case ZSTD_p_enableLongDistanceMatching:
if (cctx->cdict) return ERROR(stage_wrong);
- if (value>0) ZSTD_cLevelToCParams(cctx);
+ if (value>0)
+ ZSTD_cLevelToCCtxParams_srcSize(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); /* Optimize cParams when srcSize is known */
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_ldmHashLog:
if (cctx->cdict) return ERROR(stage_wrong);
cctx->requestedParams = *params;
-#ifdef ZSTD_MULTITHREAD
- if (cctx->mtctx)
- ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing(cctx->mtctx, params);
-#endif
-
return 0;
}
const void* dict,size_t dictSize,
ZSTD_CCtx_params params)
{
- DEBUGLOG(4, "ZSTD_compress_advanced_internal");
+ DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)",
+ (U32)srcSize);
CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
params, srcSize, ZSTDb_not_buffered) );
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
/* For use with createCCtxParams() and freeCCtxParams() only */
ZSTD_customMem customMem;
-
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
struct ZSTD_CCtx_s {
ZSTD_compressionStage_e stage;
- U32 dictID;
+ int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
ZSTD_CCtx_params requestedParams;
ZSTD_CCtx_params appliedParams;
+ U32 dictID;
void* workSpace;
size_t workSpaceSize;
size_t blockSize;
return jobParams;
}
-/*! ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing() :
- * Apply a ZSTD_CCtx_params to the compression context.
- * This entry point is accessed while compression is ongoing,
- * new parameters will be applied to next compression job.
- * However, following parameters are NOT updated :
- * - window size
- * - pledgedSrcSize
- * - nb threads
- * - job size
- * - overlap size
- */
-void ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* params)
+/*! ZSTDMT_MTCtx_updateParametersWhileCompressing() :
+ * Update compression level and parameters (except wlog)
+ * while compression is ongoing.
+ * New parameters will be applied to next compression job. */
+void ZSTDMT_MTCtx_updateParametersWhileCompressing(ZSTDMT_CCtx* mtctx, int compressionLevel, ZSTD_compressionParameters cParams)
{
- U32 const wlog = mtctx->params.cParams.windowLog;
- U32 const nbWorkers = mtctx->params.nbWorkers;
- mtctx->params = *params;
+ U32 const wlog = cParams.windowLog;
+ mtctx->params.cParams = cParams;
mtctx->params.cParams.windowLog = wlog; /* Do not modify windowLog ! Frame must keep same wlog during the whole process ! */
- mtctx->params.nbWorkers = nbWorkers; /* Do not modify nbWorkers, it must remain synchronized with CCtx Pool ! */
- /* note : other parameters not updated are simply not used beyond initialization */
+ mtctx->params.compressionLevel = compressionLevel;
}
/* ZSTDMT_getNbWorkers():
const ZSTD_CDict* cdict, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize)
{
- DEBUGLOG(2, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)",
+ DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)",
(U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx);
/* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
* Also reset jobSize and overlapLog */
size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers);
-/*! ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing() :
- * Apply a ZSTD_CCtx_params to the compression context.
- * This works even during compression, and will be applied to next compression job.
- * However, the following parameters will NOT be updated after compression has been started :
- * - window size
- * - pledgedSrcSize
- * - nb threads
- * - job size
- * - overlap size
- */
-void ZSTDMT_MTCtx_setParametersUsingCCtxParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* params);
+/*! ZSTDMT_MTCtx_updateParametersWhileCompressing() :
+ * Update compression level and parameters (except wlog)
+ * while compression is ongoing.
+ * New parameters will be applied to next compression job. */
+void ZSTDMT_MTCtx_updateParametersWhileCompressing(ZSTDMT_CCtx* mtctx, int compressionLevel, ZSTD_compressionParameters cParams);
/* ZSTDMT_getNbWorkers():
* @return nb threads currently active in mtctx.
/*! ZSTD_CCtx_setParameter() :
* Set one compression parameter, selected by enum ZSTD_cParameter.
+ * Setting a parameter is generally only possible during frame initialization (before starting compression),
+ * except for a few exceptions which can be updated during compression: compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
* Note : when `value` is an enum, cast it to unsigned for proper type checking.
- * @result : informational value (typically, the one being set, possibly corrected),
+ * @result : informational value (typically, value being set clamped correctly),
* or an error code (which can be tested with ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
-static int FIO_compressFilename_internal(cRess_t ress,
- const char* dstFileName, const char* srcFileName, int compressionLevel)
+static unsigned long long
+FIO_compressZstdFrame(const cRess_t* ressPtr,
+ const char* srcFileName, U64 fileSize,
+ int compressionLevel, U64* readsize)
{
+ cRess_t const ress = *ressPtr;
FILE* const srcFile = ress.srcFile;
FILE* const dstFile = ress.dstFile;
- U64 readsize = 0;
U64 compressedfilesize = 0;
- U64 const fileSize = UTIL_getFileSize(srcFileName);
ZSTD_EndDirective directive = ZSTD_e_continue;
- DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize);
-
- switch (g_compressionType) {
- case FIO_zstdCompression:
- break;
-
- case FIO_gzipCompression:
-#ifdef ZSTD_GZCOMPRESS
- compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
-#else
- (void)compressionLevel;
- EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n",
- srcFileName);
-#endif
- goto finish;
-
- case FIO_xzCompression:
- case FIO_lzmaCompression:
-#ifdef ZSTD_LZMACOMPRESS
- compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression);
-#else
- (void)compressionLevel;
- EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n",
- srcFileName);
-#endif
- goto finish;
-
- case FIO_lz4Compression:
-#ifdef ZSTD_LZ4COMPRESS
- compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
-#else
- (void)compressionLevel;
- EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n",
- srcFileName);
-#endif
- goto finish;
- }
+ DISPLAYLEVEL(6, "compression using zstd format \n");
/* init */
if (fileSize != UTIL_FILESIZE_UNKNOWN)
ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
+ (void)compressionLevel; (void)srcFileName;
/* Main compression loop */
do {
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
DISPLAYLEVEL(6, "fread %u bytes from source \n", (U32)inSize);
- readsize += inSize;
+ *readsize += inSize;
- if (inSize == 0 || (fileSize != UTIL_FILESIZE_UNKNOWN && readsize == fileSize))
+ if ((inSize == 0) || (*readsize == fileSize))
directive = ZSTD_e_end;
result = 1;
if (outBuff.pos) {
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos)
- EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
+ EXM_THROW(25, "Write error : cannot write compressed block");
compressedfilesize += outBuff.pos;
}
if (READY_FOR_UPDATE()) {
}
} while (directive != ZSTD_e_end);
-finish:
+ return compressedfilesize;
+}
+
+/*! FIO_compressFilename_internal() :
+ * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
+ * @return : 0 : compression completed correctly,
+ * 1 : missing or pb opening srcFileName
+ */
+static int
+FIO_compressFilename_internal(cRess_t ress,
+ const char* dstFileName, const char* srcFileName,
+ int compressionLevel)
+{
+ U64 readsize = 0;
+ U64 compressedfilesize = 0;
+ U64 const fileSize = UTIL_getFileSize(srcFileName);
+ DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize);
+
+ /* compression format selection */
+ switch (g_compressionType) {
+ default:
+ case FIO_zstdCompression:
+ compressedfilesize = FIO_compressZstdFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
+ break;
+
+ case FIO_gzipCompression:
+#ifdef ZSTD_GZCOMPRESS
+ compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
+#else
+ (void)compressionLevel;
+ EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n",
+ srcFileName);
+#endif
+ break;
+
+ case FIO_xzCompression:
+ case FIO_lzmaCompression:
+#ifdef ZSTD_LZMACOMPRESS
+ compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression);
+#else
+ (void)compressionLevel;
+ EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n",
+ srcFileName);
+#endif
+ break;
+
+ case FIO_lz4Compression:
+#ifdef ZSTD_LZ4COMPRESS
+ compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
+#else
+ (void)compressionLevel;
+ EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n",
+ srcFileName);
+#endif
+ break;
+ }
+
/* Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName,
+ DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n",
+ srcFileName,
(double)compressedfilesize / (readsize+(!readsize)/*avoid div by zero*/) * 100,
(unsigned long long)readsize, (unsigned long long) compressedfilesize,
dstFileName);
projects / thirdparty / zstd.git / commitdiff
