From 569b81adb10b3122a977ec08676b3a5a8e336774 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 Mar 2016 15:26:51 +0100 Subject: [PATCH] changed `BLOCKSIZE` into `ZSTD_BLOCKSIZE_MAX` --- lib/zbuff.c | 59 ++++++++++++++++--------------- lib/zdict.c | 8 ++--- lib/zstd_compress.c | 6 ++-- lib/zstd_decompress.c | 18 +++++----- lib/zstd_internal.h | 7 ++-- lib/zstd_static.h | 31 ++++++++-------- programs/fullbench.c | 82 ++++++++++++++++++++----------------------- 7 files changed, 104 insertions(+), 107 deletions(-) diff --git a/lib/zbuff.c b/lib/zbuff.c index a08060cf4..386b47d5a 100644 --- a/lib/zbuff.c +++ b/lib/zbuff.c @@ -40,15 +40,16 @@ ***************************************/ #include #include "error_private.h" -#include "zstd_internal.h" +#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ +#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ #include "zbuff_static.h" /* ************************************* * Constants ***************************************/ -static size_t ZBUFF_blockHeaderSize = 3; -static size_t ZBUFF_endFrameSize = 3; +static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; + /*_************************************************** * Streaming compression @@ -59,28 +60,28 @@ static size_t ZBUFF_endFrameSize = 3; * ZBUFF_CCtx objects can be reused multiple times. * * Use ZBUFF_compressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. * Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . +* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst . * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) * or an error code, which can be tested using ZBUFF_isError(). * * ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. -* Note that it will not output more than *maxDstSizePtr. +* Note that it will not output more than *dstCapacityPtr. * Therefore, some content might still be left into its internal buffer if dst buffer is too small. * @return : nb of bytes still present into internal buffer (0 if it's empty) * or an error code, which can be tested using ZBUFF_isError(). * * ZBUFF_compressEnd() instructs to finish a frame. * It will perform a flush and write frame epilogue. -* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. * @return : nb of bytes still present into internal buffer (0 if it's empty) * or an error code, which can be tested using ZBUFF_isError(). * * Hint : recommended buffer sizes (not compulsory) -* input : 128 KB block size is the internal unit, it improves latency to use this value. -* output : ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block at best speed. +* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value. +* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. * **************************************************/ typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage; @@ -137,7 +138,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dic zbc->inBuff = (char*)malloc(neededInBuffSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } - zbc->blockSize = MIN(BLOCKSIZE, zbc->inBuffSize); + zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, zbc->inBuffSize); if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; free(zbc->outBuff); /* should not be necessary */ @@ -178,7 +179,7 @@ static size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, si } static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, - void* dst, size_t* maxDstSizePtr, + void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr, int flush) /* aggregate : wait for full block before compressing */ { @@ -188,7 +189,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, const char* const iend = istart + *srcSizePtr; char* const ostart = (char*)dst; char* op = ostart; - char* const oend = ostart + *maxDstSizePtr; + char* const oend = ostart + *dstCapacityPtr; while (notDone) { switch(zbc->stage) @@ -248,7 +249,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, } *srcSizePtr = ip - istart; - *maxDstSizePtr = op - ostart; + *dstCapacityPtr = op - ostart; { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; if (hintInSize==0) hintInSize = zbc->blockSize; @@ -257,30 +258,30 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, } size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, - void* dst, size_t* maxDstSizePtr, + void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr) { - return ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, src, srcSizePtr, 0); + return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, 0); } /* *** Finalize *** */ -size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) +size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) { size_t srcSize = 0; - ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL, as some sanitizer don't like it */ + ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL, as some sanitizer don't like it */ return zbc->outBuffContentSize - zbc->outBuffFlushedSize; } -size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) +size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) { BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - BYTE* const oend = ostart + *maxDstSizePtr; - size_t outSize = *maxDstSizePtr; + BYTE* const oend = ostart + *dstCapacityPtr; + size_t outSize = *dstCapacityPtr; size_t epilogueSize, remaining; ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */ op += outSize; @@ -291,7 +292,7 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */ op += outSize; if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */ - *maxDstSizePtr = op-ostart; /* tells how many bytes were written */ + *dstCapacityPtr = op-ostart; /* tells how many bytes were written */ return remaining; } @@ -406,7 +407,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, } } /* Frame header provides buffer sizes */ - { size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ + { size_t const neededInSize = ZSTD_BLOCKSIZE_MAX; /* a block is never > ZSTD_BLOCKSIZE_MAX */ if (zbc->inBuffSize < neededInSize) { free(zbc->inBuff); zbc->inBuffSize = neededInSize; @@ -476,7 +477,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, zbc->outStart += flushedSize; if (flushedSize == toFlushSize) { zbc->stage = ZBUFFds_read; - if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize) + if (zbc->outStart + ZSTD_BLOCKSIZE_MAX > zbc->outBuffSize) zbc->outStart = zbc->outEnd = 0; break; } @@ -492,7 +493,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, *dstCapacityPtr = op-ostart; { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc); - if (nextSrcSizeHint > ZBUFF_blockHeaderSize) nextSrcSizeHint+= ZBUFF_blockHeaderSize; /* get following block header too */ + if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ nextSrcSizeHint -= zbc->inPos; /* already loaded*/ return nextSrcSizeHint; } @@ -506,7 +507,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } -size_t ZBUFF_recommendedCInSize(void) { return BLOCKSIZE; } -size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(BLOCKSIZE) + ZBUFF_blockHeaderSize + ZBUFF_endFrameSize; } -size_t ZBUFF_recommendedDInSize(void) { return BLOCKSIZE + ZBUFF_blockHeaderSize /* block header size*/ ; } -size_t ZBUFF_recommendedDOutSize(void) { return BLOCKSIZE; } +size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_MAX; } +size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; } +size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; } +size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } diff --git a/lib/zdict.c b/lib/zdict.c index 2b3d3ae8a..a7c8090a8 100644 --- a/lib/zdict.c +++ b/lib/zdict.c @@ -587,7 +587,7 @@ typedef struct { ZSTD_CCtx* ref; ZSTD_CCtx* zc; - void* workPlace; /* must be BLOCKSIZE allocated */ + void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */ } EStats_ress_t; @@ -599,9 +599,9 @@ static void ZDICT_countEStats(EStats_ress_t esr, const U32* u32Ptr; seqStore_t seqStore; - if (srcSize > BLOCKSIZE) srcSize = BLOCKSIZE; /* protection vs large samples */ + if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */ ZSTD_copyCCtx(esr.zc, esr.ref); - ZSTD_compressBlock(esr.zc, esr.workPlace, BLOCKSIZE, src, srcSize); + ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); seqStore = ZSTD_copySeqStore(esr.zc); /* count stats */ @@ -654,7 +654,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, for (u=0; u<=MaxLL; u++) litlengthCount[u]=1; esr.ref = ZSTD_createCCtx(); esr.zc = ZSTD_createCCtx(); - esr.workPlace = malloc(BLOCKSIZE); + esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); if (!esr.ref || !esr.zc || !esr.workPlace) { eSize = ERROR(memory_allocation); DISPLAYLEVEL(1, "Not enough memory"); diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 6afcddeb8..980b5c770 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -169,7 +169,7 @@ void ZSTD_validateParams(ZSTD_parameters* params) size_t ZSTD_sizeofCCtx(ZSTD_parameters params) /* hidden interface, for paramagrill */ { /* copy / pasted from ZSTD_resetCCtx_advanced */ - const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog); + const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.windowLog); const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog; const U32 divider = (params.searchLength==3) ? 3 : 4; const size_t maxNbSeq = blockSize / divider; @@ -186,7 +186,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_parameters params) /* hidden interface, for parama static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ZSTD_parameters params) { /* note : params considered validated here */ - const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog); + const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.windowLog); const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog; const U32 divider = (params.searchLength==3) ? 3 : 4; const size_t maxNbSeq = blockSize / divider; @@ -2079,7 +2079,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong); + if (srcSize > ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); zc->params.searchLength = MINMATCH; /* force ZSTD_btopt to MINMATCH in block mode */ ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.searchLength); return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0); diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 6508dbcd4..2d89a2de8 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -143,7 +143,7 @@ struct ZSTD_DCtx_s const BYTE* litPtr; size_t litBufSize; size_t litSize; - BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH]; + BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ @@ -181,7 +181,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) { memcpy(dstDCtx, srcDCtx, - sizeof(ZSTD_DCtx) - (BLOCKSIZE+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */ + sizeof(ZSTD_DCtx) - (ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */ } @@ -411,7 +411,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litCSize = ((istart[2] & 3) << 16) + (istart[3] << 8) + istart[4]; break; } - if (litSize > BLOCKSIZE) return ERROR(corruption_detected); + if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); if (HUF_isError(singleStream ? HUF_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) : @@ -419,7 +419,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+8; + dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+8; dctx->litSize = litSize; return litCSize + lhSize; } @@ -442,7 +442,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (HUF_isError(errorCode)) return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH; + dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; return litCSize + lhSize; } @@ -468,7 +468,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litSize+lhSize > srcSize) return ERROR(corruption_detected); memcpy(dctx->litBuffer, istart+lhSize, litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+8; + dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+8; dctx->litSize = litSize; return lhSize+litSize; } @@ -495,10 +495,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; break; } - if (litSize > BLOCKSIZE) return ERROR(corruption_detected); + if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); memset(dctx->litBuffer, istart[lhSize], litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH; + dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; return lhSize+1; } @@ -892,7 +892,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, const BYTE* ip = (const BYTE*)src; size_t litCSize; - if (srcSize >= BLOCKSIZE) return ERROR(srcSize_wrong); + if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->params.searchLength); diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index d67505caa..642c3777c 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -27,7 +27,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd + - zstd homepage : https://www.zstd.net */ #ifndef ZSTD_CCOMMON_H_MODULE #define ZSTD_CCOMMON_H_MODULE @@ -71,9 +71,8 @@ #define MB *(1 <<20) #define GB *(1U<<30) -#define BLOCKSIZE (128 KB) /* define, for static allocation */ - -static const size_t ZSTD_blockHeaderSize = 3; +#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */ +static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; #define BIT7 128 #define BIT6 64 diff --git a/lib/zstd_static.h b/lib/zstd_static.h index be67784ef..cb140a494 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -93,8 +93,8 @@ ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getParams() : * @return ZSTD_parameters structure for a selected compression level and srcSize. -* `srcSizeHint` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint); +* `srcSize` value is optional, select 0 if not known */ +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize); /*! ZSTD_validateParams() : * correct params value to remain within authorized range */ @@ -112,7 +112,7 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, * Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. * It avoids reloading the dictionary each time. * `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). -* Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the compression operation */ +* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, void* dst, size_t dstCapacity, @@ -124,19 +124,19 @@ ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( * Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. * It avoids reloading the dictionary each time. * `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the decompression operation */ +* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( - ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); + ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); /* ************************************** * Streaming functions (direct mode) ****************************************/ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params); +ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params); ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -187,11 +187,11 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds A ZSTD_DCtx object can be re-used multiple times. First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(). - It can provide the minimum size of buffer required to properly decompress data, + It can provide the minimum size of rolling buffer required to properly decompress data, and optionally the final size of uncompressed content. (Note : content size is an optional info that may not be present. 0 means : content size unknown) - It is done by reading a certain amount of the beginning of compressed frame. - The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max. + Frame parameters are extracted from the beginning of compressed frame. + The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work) If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result. Result : 0 when successful, it means the ZSTD_frameParams structure has been filled. >0 : means there is not enough data into `src`. Provides the expected size to successfully decode header. @@ -206,7 +206,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog). They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible. - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst'. + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity) It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. @@ -218,10 +218,10 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds * Block functions ****************************************/ /*! Block functions produce and decode raw zstd blocks, without frame metadata. - User will have to take in charge required information to regenerate data, such as block sizes. + User will have to take in charge required information to regenerate data, such as compressed and content sizes. A few rules to respect : - - Uncompressed block size must be <= 128 KB + - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB) - Compressing or decompressing requires a context structure + Use ZSTD_createCCtx() and ZSTD_createDCtx() - It is necessary to init context before starting @@ -235,6 +235,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds + ZSTD_decompressBlock() doesn't accept uncompressed data as input !! */ +#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */ size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); diff --git a/programs/fullbench.c b/programs/fullbench.c index 125dc809a..192761d85 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -97,7 +97,7 @@ #define DEFAULT_CHUNKSIZE (4<<20) #define COMPRESSIBILITY_DEFAULT 0.50 -static const size_t sampleSize = 10000000; +static const size_t g_sampleSize = 10000000; /*_************************************ @@ -256,7 +256,7 @@ size_t local_ZBUFF_compress(void* dst, size_t dstSize, void* buff2, const void* } static ZBUFF_DCtx* g_zbdc = NULL; -size_t local_ZBUFF_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) +static size_t local_ZBUFF_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { size_t srcRead = g_cSize, dstWritten = dstSize; (void)src; (void)srcSize; @@ -270,7 +270,7 @@ size_t local_ZBUFF_decompress(void* dst, size_t dstSize, void* buff2, const void /*_******************************************************* * Bench functions *********************************************************/ -size_t benchMem(void* src, size_t srcSize, U32 benchNb) +static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) { BYTE* dstBuff; size_t dstBuffSize; @@ -405,9 +405,9 @@ _cleanOut: } -int benchSample(U32 benchNb) +static int benchSample(U32 benchNb) { - size_t const benchedSize = sampleSize; + size_t const benchedSize = g_sampleSize; const char* name = "Sample 10MiB"; /* Allocation */ @@ -430,16 +430,15 @@ int benchSample(U32 benchNb) } -int benchFiles(const char** fileNamesTable, int nbFiles, U32 benchNb) +static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb) { /* Loop for each file */ - int fileIdx=0; - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + DISPLAY("Not enough memory for '%s' full size; testing %u MB only...\n", inFileName, (U32)(benchedSize>>20)); /* Alloc */ origBuff = malloc(benchedSize); @@ -458,14 +457,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles, U32 benchNb) /* Fill input buffer */ DISPLAY("Loading %s... \r", inFileName); - readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } + { + size_t readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if (readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } /* bench */ DISPLAY("\r%79s\r", ""); @@ -474,6 +473,8 @@ int benchFiles(const char** fileNamesTable, int nbFiles, U32 benchNb) benchMem(origBuff, benchedSize, benchNb); else for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb); + + free(origBuff); } return 0; @@ -489,8 +490,9 @@ static int usage(const char* exename) return 0; } -static int usage_advanced(void) +static int usage_advanced(const char* exename) { + usage(exename); DISPLAY( "\nAdvanced options :\n"); DISPLAY( " -b# : test only function # \n"); DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); @@ -502,41 +504,34 @@ static int badusage(const char* exename) { DISPLAY("Wrong parameters\n"); usage(exename); - return 0; + return 1; } int main(int argc, const char** argv) { - int i, - filenamesStart=0, - result; - const char* exename=argv[0]; - const char* input_filename=0; + int i, filenamesStart=0, result; + const char* exename = argv[0]; + const char* input_filename = NULL; U32 benchNb = 0, main_pause = 0; - // Welcome message DISPLAY(WELCOME_MESSAGE); + if (argc<1) return badusage(exename); - if (argc<1) { badusage(exename); return 1; } - - for(i=1; i= '0') && (argument[1]<= '9')) { @@ -573,7 +568,7 @@ int main(int argc, const char** argv) break; /* Unknown command */ - default : badusage(exename); return 1; + default : return badusage(exename); } } continue; @@ -583,9 +578,10 @@ int main(int argc, const char** argv) if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } } - if (filenamesStart==0) + if (filenamesStart==0) /* no input file */ result = benchSample(benchNb); - else result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb); + else + result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb); if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } -- 2.47.2