From: Yann Collet Date: Wed, 3 Feb 2016 01:11:32 +0000 (+0100) Subject: move _usingDict() to stable API (zstd.h) X-Git-Tag: v0.5.0~1^2~3^2~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7d968c7bdd33606af33fdbfbbfafacb6a9fd2c4b;p=thirdparty%2Fzstd.git move _usingDict() to stable API (zstd.h) --- diff --git a/lib/zstd.h b/lib/zstd.h index d74f56a35..652609f36 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -70,27 +70,26 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); /* ************************************* * Simple functions ***************************************/ -ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t maxDstSize, +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t maxOriginalSize, +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); /** ZSTD_compress() : - Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'. + Compresses @srcSize bytes from buffer @src into buffer @dst of size @dstCapacity. Destination buffer must be already allocated. - Compression runs faster if maxDstSize >= ZSTD_compressBound(srcSize). - return : the number of bytes written into buffer 'dst' - or an error code if it fails (which can be tested using ZSTD_isError()) + Compression runs faster if @dstCapacity >= ZSTD_compressBound(srcSize). + @return : the number of bytes written into @dst + or an error code if it fails (which can be tested using ZSTD_isError()) ZSTD_decompress() : - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (<= maxOriginalSize) - or an errorCode if it fails (which can be tested using ZSTD_isError()) + @compressedSize : is the _exact_ size of the compressed blob (or decompression will fail) + @dst must be large enough, equal or larger than originalSize. + @return : the number of bytes decompressed into @dst (<= @dstCapacity) + or an errorCode if it fails (which can be tested using ZSTD_isError()) */ @@ -105,7 +104,7 @@ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /** provides error co /* ************************************* -* Advanced functions +* Explicit memory management ***************************************/ /** Compression context management */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */ @@ -114,7 +113,7 @@ ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /** ZSTD_compressCCtx() : Same as ZSTD_compress(), but requires an already allocated ZSTD_CCtx (see ZSTD_createCCtx()) */ -ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); /** Decompression context management */ typedef struct ZSTD_DCtx_s ZSTD_DCtx; @@ -123,7 +122,30 @@ ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /** ZSTD_decompressDCtx * Same as ZSTD_decompress(), but requires an already allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ -ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +/*-*********************** +* Dictionary API +*************************/ + +/*! ZSTD_compress_usingDict +* Compression using a pre-defined Dictionary content (see dictBuilder) +* Note : dict can be NULL, in which case, it's equivalent to ZSTD_compressCCtx() */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict +* Decompression using a pre-defined Dictionary content (see dictBuilder) +* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted +* Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); #if defined (__cplusplus) diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index 1e8830b1e..8aa37650e 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -71,9 +71,9 @@ ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); -ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); -ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); +ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); +ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); +ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); /** ************************************************ * Streaming compression @@ -87,24 +87,24 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDst * Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary. * * Use ZBUFF_compressContinue() repetitively to consume input stream. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr. * Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or move dst . -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) * or an error code, which can be tested using ZBUFF_isError(). * * At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush(). -* The nb of bytes written into `dst` will be reported into *maxDstSizePtr. -* Note that the function cannot output more than the size of `dst` buffer (initial value of *maxDstSizePtr). -* Therefore, some content might still be left into internal buffer if dst buffer is too small. +* The nb of bytes written into @dst will be reported into *dstCapacityPtr. +* Note that the function cannot output more than *dstCapacityPtr, +* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small. * @return : nb of bytes still present into internal buffer (0 if it's empty) * or an error code, which can be tested using ZBUFF_isError(). * * ZBUFF_compressEnd() instructs to finish a frame. * It will perform a flush and write frame epilogue. -* Note that the epilogue is necessary for decoders to consider a frame completed. -* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. * In which case, call again ZBUFF_compressFlush() to complete the flush. * @return : nb of bytes still present into internal buffer (0 if it's empty) * or an error code, which can be tested using ZBUFF_isError(). @@ -112,7 +112,7 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDst * Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedCInSize / ZBUFF_recommendedCOutSize * input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, it improves latency to use this value (skipped buffering). * output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering. -* By using both, you ensure that input will be entirely consumed, and output will always contain the result. +* By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering. * **************************************************/ @@ -123,7 +123,7 @@ ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); +ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); /** ************************************************ * Streaming decompression @@ -135,10 +135,10 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* * Note that ZBUFF_DCtx objects can be reused multiple times. * * Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst. +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) * or 0 when a frame is completely decoded * or an error code, which can be tested using ZBUFF_isError(). diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index c89d5ca1b..6684bae46 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -69,8 +69,6 @@ static const U32 g_searchStrength = 8; /* ************************************* * Helper functions ***************************************/ -unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } - size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } @@ -2262,6 +2260,9 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi /*- Pre-defined compression levels -*/ +#define ZSTD_MAX_CLEVEL 20 +unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } + static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ /* W, C, H, S, L, strat */ diff --git a/lib/zstd_static.h b/lib/zstd_static.h index de2212d9d..50f301faa 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -82,6 +82,8 @@ typedef struct /* ************************************* * Advanced functions ***************************************/ +ZSTDLIB_API unsigned ZSTD_maxCLevel (void); + /*! ZSTD_getParams * @return ZSTD_parameters structure for a selected compression level and srcSize. * @srcSizeHint value is optional, select 0 if not known */ @@ -91,19 +93,10 @@ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint * correct params value to remain within authorized range */ ZSTDLIB_API void ZSTD_validateParams(ZSTD_parameters* params); -/*! ZSTD_compress_usingDict -* Same as ZSTD_compressCCtx(), loading a Dictionary content. -* Note : dict can be NULL, in which case, it's equivalent to ZSTD_compressCCtx() */ -ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - int compressionLevel); - /*! ZSTD_compress_advanced * Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, ZSTD_parameters params); @@ -115,19 +108,11 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, * Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the compression operation */ ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize); /*- Advanced Decompression functions -*/ -/*! ZSTD_decompress_usingDict -* Same as ZSTD_decompressDCtx, using a Dictionary content as prefix -* Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */ -ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - /*! ZSTD_decompress_usingPreparedDCtx * Same as ZSTD_decompress_usingDict, but using a reference context preparedDCtx, where dictionary has been loaded. * It avoids reloading the dictionary each time. @@ -135,7 +120,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, * Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the decompression operation */ ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -147,8 +132,8 @@ ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dic ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params); ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); +ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity); /** Streaming compression, synchronous mode (bufferless) @@ -181,7 +166,7 @@ ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /** Streaming decompression, direct mode (bufferless) @@ -217,34 +202,26 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma /* ************************************** * Block functions ****************************************/ -/*!Block functions produce and decode raw zstd blocks, without frame metadata. - Frame headers won't be generated. - User will have to save and regenerate fields required to regenerate data, such as block sizes. - - A few rules to respect : - - Uncompressed block size must be <= 128 KB - - Compressing or decompressing requires a context structure - + Use ZSTD_createXCtx() to create them - - It is necessary to init context before starting - + compression : ZSTD_compressBegin() - + decompression : ZSTD_decompressBegin() - + variants _usingDict() are also allowed - + copyXCtx() works too - - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. - In which case, nothing is produced into `dst`. - + User must test for such outcome and deal directly with uncompressed data - + ZSTD_decompressBlock() doesn't accept uncompressed data as input !! +/*! Block functions produce and decode raw zstd blocks, without frame metadata. + User will have to save and regenerate necessary information to regenerate data, such as block sizes. + + A few rules to respect : + - Uncompressed block size must be <= 128 KB + - Compressing or decompressing requires a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : ZSTD_compressBegin() + + decompression : ZSTD_decompressBegin() + + variants _usingDict() are also allowed + + copyCCtx() and copyDCtx() work too + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. + In which case, nothing is produced into @dst. + + User must test for such outcome and deal directly with uncompressed data + + ZSTD_decompressBlock() doesn't accept uncompressed data as input !! */ -size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); - - -/* ************************************* -* Pre-defined compression levels -***************************************/ -#define ZSTD_MAX_CLEVEL 20 -ZSTDLIB_API unsigned ZSTD_maxCLevel (void); +size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /* *************************************