From: Yann Collet Date: Fri, 11 Mar 2016 20:58:04 +0000 (+0100) Subject: created ZSTD_frameParams X-Git-Tag: v0.6.0^2~17^2~68^2~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0e491c01fefc00f82f44d31508285b96f128d310;p=thirdparty%2Fzstd.git created ZSTD_frameParams --- diff --git a/lib/zbuff.c b/lib/zbuff.c index 4c1eb2cf2..69341df8e 100644 --- a/lib/zbuff.c +++ b/lib/zbuff.c @@ -298,28 +298,29 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) } - -/** ************************************************ -* Streaming decompression +/*-*************************************************************************** +* Streaming decompression howto * -* A ZBUFF_DCtx object is required to track streaming operation. +* A ZBUFF_DCtx object is required to track streaming operations. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation. -* ZBUFF_DCtx objects can be reused multiple times. +* Use ZBUFF_decompressInit() to start a new decompression operation, +* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFF_DCtx objects can be re-init multiple times. * * Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. -* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) -* or 0 when a frame is completely decoded +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, * or an error code, which can be tested using ZBUFF_isError(). * -* Hint : recommended buffer sizes (not compulsory) -* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. -* input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* **************************************************/ +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() +* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader, ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; @@ -329,7 +330,7 @@ typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_dec #define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */ struct ZBUFF_DCtx_s { ZSTD_DCtx* zc; - ZSTD_parameters params; + ZSTD_frameParams fParams; char* inBuff; size_t inBuffSize; size_t inPos; @@ -381,14 +382,16 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) /* *** Decompression *** */ -size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr) +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) { const char* const istart = (const char*)src; const char* ip = istart; const char* const iend = istart + *srcSizePtr; char* const ostart = (char*)dst; char* op = ostart; - char* const oend = ostart + *maxDstSizePtr; + char* const oend = ostart + *dstCapacityPtr; U32 notDone = 1; while (notDone) { @@ -400,13 +403,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt case ZBUFFds_readHeader : /* read header from src */ { - size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr); + size_t headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr); if (ZSTD_isError(headerSize)) return headerSize; if (headerSize) { /* not enough input to decode header : tell how many bytes would be necessary */ memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr); zbc->hPos += *srcSizePtr; - *maxDstSizePtr = 0; + *dstCapacityPtr = 0; zbc->stage = ZBUFFds_loadHeader; return headerSize - zbc->hPos; } @@ -422,11 +425,11 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt src, *srcSizePtr); zbc->hPos += headerSize; ip += headerSize; - headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos); + headerSize = ZSTD_getFrameParams(&(zbc->fParams), zbc->headerBuffer, zbc->hPos); if (ZSTD_isError(headerSize)) return headerSize; if (headerSize) { /* not enough input to decode header : tell how many bytes would be necessary */ - *maxDstSizePtr = 0; + *dstCapacityPtr = 0; return headerSize - zbc->hPos; } // zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */ @@ -435,7 +438,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt case ZBUFFds_decodeHeader: /* apply header to create / resize buffers */ { - size_t neededOutSize = (size_t)1 << zbc->params.windowLog; + size_t neededOutSize = (size_t)1 << zbc->fParams.windowLog; size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ if (zbc->inBuffSize < neededInSize) { free(zbc->inBuff); @@ -524,7 +527,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt } } *srcSizePtr = ip-istart; - *maxDstSizePtr = op-ostart; + *dstCapacityPtr = op-ostart; { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc); diff --git a/lib/zbuff.h b/lib/zbuff.h index d3275b7df..7a914a4a7 100644 --- a/lib/zbuff.h +++ b/lib/zbuff.h @@ -75,7 +75,7 @@ ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstC ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); /*-************************************************* -* Streaming compression +* Streaming compression - howto * * A ZBUFF_CCtx object is required to track streaming operation. * Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. @@ -127,26 +127,27 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, const void* src, size_t* srcSizePtr); /*-*************************************************************************** -* Streaming decompression +* Streaming decompression howto * * A ZBUFF_DCtx object is required to track streaming operations. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. * Use ZBUFF_decompressInit() to start a new decompression operation, * or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFF_DCtx objects can be reused multiple times. +* Note that ZBUFF_DCtx objects can be re-init multiple times. * * Use ZBUFF_decompressContinue() repetitively to consume your input. * *srcSizePtr and *dstCapacityPtr can be any size. * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) -* or 0 when a frame is completely decoded +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, * or an error code, which can be tested using ZBUFF_isError(). * -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() / ZBUFF_recommendedDOutSize() +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() * output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * *******************************************************************************/ diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index fc88eb168..c1ab910bc 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -103,7 +103,7 @@ struct ZSTD_CCtx_s size_t workSpaceSize; size_t blockSize; size_t hbSize; - char headerBuffer[ZSTD_frameHeaderSize_max]; + char headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 60136b615..2625f5476 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -136,7 +136,7 @@ struct ZSTD_DCtx_s const void* dictEnd; size_t expected; size_t headerSize; - ZSTD_parameters params; + ZSTD_frameParams fParams; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; U32 flagStaticTables; @@ -144,7 +144,7 @@ struct ZSTD_DCtx_s size_t litBufSize; size_t litSize; BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_frameHeaderSize_max]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); } @@ -159,7 +159,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->dictEnd = NULL; dctx->hufTableX4[0] = HufLog; dctx->flagStaticTables = 0; - dctx->params.searchLength = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ + dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength); return 0; } @@ -286,15 +286,15 @@ static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_ } -size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize) +size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize) { U32 magicNumber; if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max; magicNumber = MEM_readLE32(src); if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); - memset(params, 0, sizeof(*params)); - params->windowLog = (((const BYTE*)src)[4] & 15) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - params->searchLength = (((const BYTE*)src)[4] & 16) ? MINMATCH-1 : MINMATCH; + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); + fparamsPtr->windowLog = (((const BYTE*)src)[4] & 15) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + fparamsPtr->mml = (((const BYTE*)src)[4] & 16) ? MINMATCH-1 : MINMATCH; if ((((const BYTE*)src)[4] >> 5) != 0) return ERROR(frameParameter_unsupported); /* reserved 3 bits */ return 0; } @@ -308,8 +308,8 @@ static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_ size_t result; if (srcSize != zc->headerSize) return ERROR(srcSize_wrong); - result = ZSTD_getFrameParams(&(zc->params), src, srcSize); - if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); + result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize); + if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); return result; } @@ -792,7 +792,7 @@ static size_t ZSTD_decompressSequences( const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - const U32 mls = dctx->params.searchLength; + const U32 mls = dctx->fParams.mml; /* Build Decoding Tables */ errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, @@ -885,9 +885,9 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } -/*! ZSTD_decompress_continueDCtx -* dctx must have been properly initialized */ -static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx, +/*! ZSTD_decompress_continueDCtx() : +* `dctx` must have been properly initialized */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { @@ -965,7 +965,7 @@ size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDC { ZSTD_copyDCtx(dctx, refDCtx); ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); + return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize); } @@ -977,7 +977,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength); ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); + return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize); } @@ -986,6 +986,7 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const return ZSTD_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0); } + size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) { #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1) diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 409e5e57a..d67505caa 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -71,11 +71,9 @@ #define MB *(1 <<20) #define GB *(1U<<30) -#define BLOCKSIZE (128 KB) /* define, for static allocation */ +#define BLOCKSIZE (128 KB) /* define, for static allocation */ static const size_t ZSTD_blockHeaderSize = 3; -static const size_t ZSTD_frameHeaderSize_min = 5; -#define ZSTD_frameHeaderSize_max 5 /* define, for static allocation */ #define BIT7 128 #define BIT6 64 diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 4f23fca98..e3c0dfc45 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -165,13 +165,17 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci You can then reuse ZSTD_CCtx to compress some new frame. */ +typedef struct { U64 frameSize; U32 windowLog; U32 mml; } ZSTD_frameParams; + +#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ +static const size_t ZSTD_frameHeaderSize_min = 5; +static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); - ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -182,15 +186,16 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. A ZSTD_DCtx object can be re-used multiple times. - First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams(). - This operation is independent, and just needs enough input data to properly decode the frame header. - Objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding. - Result : 0 when successful, it means the ZSTD_parameters structure has been filled. - >0 : means there is not enough data into src. Provides the expected size to successfully decode header. + First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(). + It requires to read the beginning of compressed frame. + The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max. + If you don't provide enough length, function will return the minimum size it wants to produce a result. + Result : 0 when successful, it means the ZSTD_frameParams structure has been filled. + >0 : means there is not enough data into src. Provides the expected size to successfully decode header. errorCode, which can be tested using ZSTD_isError() - Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict() - Alternatively, you can copy a prepared context, using ZSTD_copyDCtx() + Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict(). + Alternatively, you can copy a prepared context, using ZSTD_copyDCtx(). Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().