ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
{
- DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize);
+ DEBUGLOG(4, " setting pledgedSrcSize to %u", (U32)pledgedSrcSize);
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
return 0;
{
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */
- DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
+ DEBUGLOG(4, "load dictionary of size %u", (U32)dictSize);
ZSTD_freeCDict(cctx->cdictLocal); /* in case one already exists */
if (dict==NULL || dictSize==0) { /* no dictionary mode */
cctx->cdictLocal = NULL;
ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams)
{
- DEBUGLOG(3, "ZSTD_initCDict_internal, mode %u", (U32)dictMode);
+ DEBUGLOG(4, "ZSTD_initCDict_internal, mode %u", (U32)dictMode);
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
cdict->dictBuffer = NULL;
cdict->dictContent = dictBuffer;
ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
{
- DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
+ DEBUGLOG(4, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp)
{
- DEBUGLOG(5, "ZSTD_compress_generic, endOp=%u ", (U32)endOp);
+ DEBUGLOG(5, "ZSTD_compress_generic");
/* check conditions */
if (output->pos > output->size) return ERROR(GENERIC);
if (input->pos > input->size) return ERROR(GENERIC);
#ifdef ZSTD_MULTITHREAD
if (cctx->appliedParams.nbThreads > 1) {
size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
+ DEBUGLOG(5, "ZSTDMT_compressStream_generic result : %u", (U32)flushMin);
if ( ZSTD_isError(flushMin)
|| (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
ZSTD_startNewCompression(cctx);
typedef struct {
buffer_t src;
const void* srcStart;
- size_t prefixSize;
+ size_t dictSize;
size_t srcSize;
buffer_t dstBuff;
size_t cSize;
{
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool);
- const void* const src = (const char*)job->srcStart + job->prefixSize;
+ const void* const src = (const char*)job->srcStart + job->dictSize;
buffer_t dstBuff = job->dstBuff;
- DEBUGLOG(5, "ZSTDMT_compressChunk: job (first:%u) (last:%u) : prefixSize %u, srcSize %u ",
- job->firstChunk, job->lastChunk, (U32)job->prefixSize, (U32)job->srcSize);
+ DEBUGLOG(5, "ZSTDMT_compressChunk: job (first:%u) (last:%u) : dictSize %u, srcSize %u",
+ job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
if (cctx==NULL) {
job->cSize = ERROR(memory_allocation);
if (job->cdict) {
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
DEBUGLOG(4, "ZSTDMT_compressChunk: init using CDict");
- assert(job->firstChunk); /* only allowed for first job */
+ assert(job->firstChunk); /* should only happen for first segment */
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
} else { /* srcStart points at reloaded section */
ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */
size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN;
/* load dictionary in "content-only" mode (no header analysis) */
- size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->prefixSize, ZSTD_dm_rawContent, jobParams, pledgedSrcSize);
- DEBUGLOG(5, "ZSTD_compressBegin_advanced_internal called with windowLog = %u ", jobParams.cParams.windowLog);
+ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, pledgedSrcSize);
if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
job->cSize = initError;
goto _endJob;
}
}
- if (!job->firstChunk) { /* flush and overwrite frame header when it's not first job */
+ if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */
size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
ZSTD_invalidateRepCodes(cctx);
job->cSize = (job->lastChunk) ?
ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
- DEBUGLOG(5, "compressed %u bytes into %u bytes (first:%u) (last:%u) ",
+ DEBUGLOG(5, "compressed %u bytes into %u bytes (first:%u) (last:%u)",
(unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
- DEBUGLOG(5, "dstBuff.size : %u ; => %s ", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
+ DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
_endJob:
ZSTDMT_releaseCCtx(job->cctxPool, cctx);
size_t const overlapSize = (overlapRLog>=9) ? 0 : (size_t)1 << (params.cParams.windowLog - overlapRLog);
unsigned nbChunks = computeNbChunks(srcSize, params.cParams.windowLog, params.nbThreads);
size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks;
- size_t const avgChunkSize = (((proposedChunkSize-1) & 0x1FFFF) < 0x7FFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */
+ size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0x7FFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */
const char* const srcStart = (const char*)src;
size_t remainingSrcSize = srcSize;
unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize)); /* presumes avgChunkSize >= 256 KB, which should be the case */
assert(mtctx->cctxPool->totalCCtx == params.nbThreads);
DEBUGLOG(4, "ZSTDMT_compress_advanced_internal");
- DEBUGLOG(4, "nbChunks : %2u (raw chunkSize : %u bytes; fixed chunkSize: %u) ",
- nbChunks, (U32)proposedChunkSize, (U32)avgChunkSize);
+ DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
if (nbChunks==1) { /* fallback to single-thread mode */
ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams);
mtctx->jobs[u].src = g_nullBuffer;
mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
- mtctx->jobs[u].prefixSize = dictSize;
+ mtctx->jobs[u].dictSize = dictSize;
mtctx->jobs[u].srcSize = chunkSize;
mtctx->jobs[u].cdict = (u==0) ? cdict : NULL;
mtctx->jobs[u].fullFrameSize = srcSize;
zcs->jobs[jobID].src = zcs->inBuff.buffer;
zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start;
zcs->jobs[jobID].srcSize = srcSize;
- zcs->jobs[jobID].prefixSize = zcs->dictSize;
+ zcs->jobs[jobID].dictSize = zcs->dictSize;
assert(zcs->inBuff.filled >= srcSize + zcs->dictSize);
zcs->jobs[jobID].params = zcs->params;
/* do not calculate checksum within sections, but write it in header for first section */
{
size_t const newJobThreshold = mtctx->dictSize + mtctx->targetSectionSize;
unsigned forwardInputProgress = 0;
- DEBUGLOG(5, "ZSTDMT_compressStream_generic ");
+ DEBUGLOG(5, "ZSTDMT_compressStream_generic");
assert(output->pos <= output->size);
assert(input->pos <= input->size);
return ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
}
#endif
- return hSize; /* error */
+ return hSize; /* error */
}
if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
- size_t const remainingInput = (size_t)(iend-ip);
- assert(iend >= ip);
- if (toLoad > remainingInput) { /* not enough input to load full header */
- if (remainingInput > 0) {
- memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
- zds->lhSize += remainingInput;
+ if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
+ if (iend-ip > 0) {
+ memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
+ zds->lhSize += iend-ip;
}
input->pos = input->size;
return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
if (cSize <= (size_t)(iend-istart)) {
- /* shortcut : using single-pass mode */
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict);
if (ZSTD_isError(decompressedSize)) return decompressedSize;
- DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
ip = istart + cSize;
op += decompressedSize;
zds->expected = 0;
}
/* control buffer memory usage */
- DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)",
- (U32)(zds->fParams.windowSize >>10),
- (U32)(zds->maxWindowSize >> 10) );
+ DEBUGLOG(4, "Control max buffer memory usage (max %u KB)",
+ (U32)(zds->maxWindowSize >> 10));
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
* More threads improve speed, but also increase memory usage.
* Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
* Special: value 0 means "do not change nbThreads" */
- ZSTD_p_jobSize, /* Size of a compression job. This value is only enforced in streaming (non-blocking) mode.
- * Each compression job is completed in parallel, so indirectly controls the nb of active threads.
+ ZSTD_p_jobSize, /* Size of a compression job. Each compression job is completed in parallel.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
* The minimum size is automatically and transparently enforced */
* - Compression parameters cannot be changed once compression is started.
* - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
* - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- * - In single-thread mode (default), function is blocking : it completed its job before returning to caller.
- * - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
- * and then immediately returns, just indicating that there is some data remaining to be flushed.
- * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- * - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
- * - @return provides the minimum amount of data remaining to be flushed from internal buffers
+ * - @return provides the minimum amount of data still to flush from internal buffers
* or an error code, which can be tested using ZSTD_isError().
- * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
- * This is useful to determine if a ZSTD_e_flush or ZSTD_e_end directive is completed.
- * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
+ * if @return != 0, flush is not fully completed, there is some data left within internal buffers.
+ * - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
* only ZSTD_e_end or ZSTD_e_flush operations are allowed.
- * Before starting a new compression job, or changing compression parameters,
- * it is required to fully flush internal buffers.
+ * It is necessary to fully flush internal buffers
+ * before starting a new compression job, or changing compression parameters.
*/
ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
ZSTD_outBuffer* output,
{ ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
if (ZSTD_isError(initError)) goto _output_error;
- outBuff.dst = decodedBuffer;
- outBuff.size = CNBufferSize;
- outBuff.pos = 0;
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
+ outBuff.dst = decodedBuffer;
+ outBuff.size = CNBufferSize;
+ outBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000) ); /* too small limit */
- outBuff.dst = decodedBuffer;
- outBuff.size = CNBufferSize;
- outBuff.pos = 0;
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
+ outBuff.dst = decodedBuffer;
+ outBuff.size = CNBufferSize;
+ outBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
params.fParams.contentSizeFlag = 1;
CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
} /* cstream advanced shall write content size = 0 */
- outBuff.dst = compressedBuffer;
- outBuff.size = compressedBufferSize;
- outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = 0;
inBuff.pos = 0;
+ outBuff.dst = compressedBuffer;
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
cSize = outBuff.pos;
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
- outBuff.dst = compressedBuffer;
- outBuff.size = compressedBufferSize;
- outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = 0;
inBuff.pos = 0;
+ outBuff.dst = compressedBuffer;
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
cSize = outBuff.pos;
{ ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) );
}
- outBuff.dst = compressedBuffer;
+ outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
if (r != 0) goto _output_error; } /* error, or some data not flushed */
DISPLAYLEVEL(3, "OK \n");
- /* Complex multithreading + dictionary test */
- { U32 const nbThreads = 2;
- size_t const jobSize = 4 * 1 MB;
- size_t const srcSize = jobSize * nbThreads; /* we want each job to have predictable size */
- size_t const segLength = 2 KB;
- size_t const offset = 600 KB; /* must be larger than window defined in cdict */
- size_t const start = jobSize + (offset-1);
- const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
- BYTE* const dst = (BYTE*)CNBuffer + start - offset;
- DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (U32)srcSize);
- ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, 3);
- ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, 2);
- ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, jobSize);
- assert(start > offset);
- assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
- memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
- outBuff.dst = compressedBuffer;
- outBuff.size = compressedBufferSize;
- outBuff.pos = 0;
- inBuff.src = CNBuffer;
- inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
- inBuff.pos = 0;
- }
- { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */
- ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
- DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
- ZSTD_CCtx_refCDict(zc, cdict);
- CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
- ZSTD_CCtx_refCDict(zc, NULL); /* do not keep a reference to cdict, as its lifetime ends */
- ZSTD_freeCDict(cdict);
- }
- if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
- cSize = outBuff.pos;
- DISPLAYLEVEL(3, "OK \n");
-
- DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
- { ZSTD_DStream* const dstream = ZSTD_createDCtx();
- ZSTD_frameHeader zfh;
- ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
- DISPLAYLEVEL(5, "frame windowsize = %u : ", (U32)zfh.windowSize);
- outBuff.dst = decodedBuffer;
- outBuff.size = CNBufferSize;
- outBuff.pos = 0;
- inBuff.src = compressedBuffer;
- inBuff.pos = 0;
- CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
- inBuff.size = 1; /* avoid shortcut to single-pass mode */
- CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
- inBuff.size = cSize;
- CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
- if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
- ZSTD_freeDStream(dstream);
- }
- DISPLAYLEVEL(3, "OK \n");
-
DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
{ unsigned const kMaxWindowLog = 24;
unsigned value;
projects / thirdparty / zstd.git / commitdiff
