From: Yann Collet Date: Wed, 25 Nov 2015 13:42:45 +0000 (+0100) Subject: Added : zstd buffered API X-Git-Tag: zstd-0.4.0^2~23^2~11 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=88fcd2916e7e1f37a468df8a64b2388f8eb41aa9;p=thirdparty%2Fzstd.git Added : zstd buffered API --- diff --git a/Makefile b/Makefile index 7caaa4661..4ec4a7954 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,14 @@ export VERSION := 0.4.0 PRGDIR = programs ZSTDDIR = lib -.PHONY: clean +# Define nul output +ifneq (,$(filter Windows%,$(OS))) +VOID = nul +else +VOID = /dev/null +endif + +.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan default: zstdprogram @@ -49,8 +56,8 @@ zstdprogram: $(MAKE) -C $(PRGDIR) clean: - $(MAKE) -C $(ZSTDDIR) $@ - $(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) + @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @echo Cleaning completed diff --git a/NEWS b/NEWS index c8177e8da..75c9b87e3 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ v0.4.0 +Command line utility is now compatible with high compression levels Removed zstdhc => merged into zstd +Added : ZBUFF API (see zstd_buffered.h) Rolling buffer support v0.3.6 diff --git a/lib/error.h b/lib/error.h index 68e8d46c3..86f3c384f 100644 --- a/lib/error.h +++ b/lib/error.h @@ -68,7 +68,7 @@ extern "C" { #define ERROR_LIST(ITEM) \ ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \ - ITEM(PREFIX(mode_unsupported)) \ + ITEM(PREFIX(mode_unsupported)) ITEM(PREFIX(init_missing))\ ITEM(PREFIX(memory_allocation)) \ ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \ ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \ diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c new file mode 100644 index 000000000..28bff9ec3 --- /dev/null +++ b/lib/zstd_buffered.c @@ -0,0 +1,531 @@ +/* + Buffered version of Zstd compression library + Copyright (C) 2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd source repository : https://github.com/Cyan4973/zstd + - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* The objects defined into this file should be considered experimental. + * They are not labelled stable, as their prototype may change in the future. + * You can use them for tests, provide feedback, or if you can endure risk of future changes. + */ + +/* ************************************* +* Includes +***************************************/ +#include +#include "error.h" +#include "zstd_static.h" +#include "zstd_buffered_static.h" + + +/** ************************************************ +* Streaming compression +* +* A ZBUFF_CCtx object is required to track streaming operation. +* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. +* Use ZBUFF_compressInit() to start a new compression operation. +* ZBUFF_CCtx objects can be reused multiple times. +* +* Use ZBUFF_compressContinue() repetitively to consume your input. +* *srcSizePtr and *maxDstSizePtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. +* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. +* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. +* Note that it will not output more than *maxDstSizePtr. +* Therefore, some content might still be left into its internal buffer if dst buffer is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressEnd() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) +* input : 128 KB block size is the internal unit, it improves latency to use this value. +* output : ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block at best speed. +* **************************************************/ + +typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage; + +/* *** Ressources *** */ +struct ZBUFF_CCtx_s { + ZSTD_CCtx* zc; + char* inBuff; + size_t inBuffSize; + size_t inToCompress; + size_t inBuffPos; + size_t inBuffTarget; + size_t blockSize; + char* outBuff; + size_t outBuffSize; + size_t outBuffContentSize; + size_t outBuffFlushedSize; + ZBUFF_cStage stage; +}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ + +ZBUFF_CCtx* ZBUFF_createCCtx(void) +{ + ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx)); + memset(zbc, 0, sizeof(*zbc)); + zbc->zc = ZSTD_createCCtx(); + return zbc; +} + +size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) +{ + if (zbc==NULL) return 0; /* support free on NULL */ + ZSTD_freeCCtx(zbc->zc); + free(zbc); + return 0; +} + + +/* *** Initialization *** */ + +#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) ) +#define BLOCKSIZE (128 * 1024) /* a bit too "magic", should come from reference */ +size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, ZSTD_parameters params) +{ + size_t neededInBuffSize; + + ZSTD_validateParams(¶ms); + neededInBuffSize = (size_t)1 << params.windowLog; + + /* allocate buffers */ + if (zbc->inBuffSize < neededInBuffSize) + { + zbc->inBuffSize = neededInBuffSize; + free(zbc->inBuff); /* should not be necessary */ + zbc->inBuff = (char*)malloc(neededInBuffSize); + if (zbc->inBuff == NULL) return ERROR(memory_allocation); + } + zbc->blockSize = MIN(BLOCKSIZE, zbc->inBuffSize); + if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) + { + zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; + free(zbc->outBuff); /* should not be necessary */ + zbc->outBuff = (char*)malloc(zbc->outBuffSize); + if (zbc->outBuff == NULL) return ERROR(memory_allocation); + } + + zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, zbc->outBuff, zbc->outBuffSize, params); + if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize; + + zbc->inToCompress = 0; + zbc->inBuffPos = 0; + zbc->inBuffTarget = zbc->blockSize; + zbc->outBuffFlushedSize = 0; + zbc->stage = ZBUFFcs_flush; /* starts by flushing the header */ + return 0; /* ready to go */ +} + +size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) +{ + return ZBUFF_compressInit_advanced(zbc, ZSTD_getParams(compressionLevel, 0)); +} + + + +/* *** Compression *** */ + +static size_t ZBUFF_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + size_t length = MIN(maxDstSize, srcSize); + memcpy(dst, src, length); + return length; +} + +static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, + void* dst, size_t* maxDstSizePtr, + const void* src, size_t* srcSizePtr, + int flush) /* aggregate : wait for full block before compressing */ +{ + U32 notDone = 1; + const char* const istart = (const char*)src; + const char* ip = istart; + const char* const iend = istart + *srcSizePtr; + char* const ostart = (char*)dst; + char* op = ostart; + char* const oend = ostart + *maxDstSizePtr; + + while (notDone) + { + switch(zbc->stage) + { + case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ + + case ZBUFFcs_load: + /* complete inBuffer */ + { + size_t toLoad = zbc->inBuffTarget - zbc->inBuffPos; + size_t loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip); + zbc->inBuffPos += loaded; + ip += loaded; + if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) + { notDone = 0; break; } /* not enough input to get a full block : stop there, wait for more */ + } + /* compress current block (note : this stage cannot be stopped in the middle) */ + { + void* cDst; + size_t cSize; + size_t iSize = zbc->inBuffPos - zbc->inToCompress; + if ((size_t)(oend-op) > ZSTD_compressBound(iSize)) + cDst = op; /* compress directly into output buffer (avoid flush stage) */ + else + cDst = zbc->outBuff; + cSize = ZSTD_compressContinue(zbc->zc, cDst, oend-op, zbc->inBuff + zbc->inToCompress, iSize); + if (ZSTD_isError(cSize)) return cSize; + /* prepare next block */ + zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize; + if (zbc->inBuffTarget > zbc->inBuffSize) + { zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; } + zbc->inToCompress = zbc->inBuffPos; + if (cDst == op) { op += cSize; break; } /* no need to flush */ + zbc->outBuffContentSize = cSize; + zbc->outBuffFlushedSize = 0; + zbc->stage = ZBUFFcs_flush; + // break; /* flush stage follows */ + } + + case ZBUFFcs_flush: + /* flush into dst */ + { + size_t toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; + size_t flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); + op += flushed; + zbc->outBuffFlushedSize += flushed; + if (toFlush!=flushed) + { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */ + zbc->outBuffContentSize = 0; + zbc->outBuffFlushedSize = 0; + zbc->stage = ZBUFFcs_load; + break; + } + } + } + + *srcSizePtr = ip - istart; + *maxDstSizePtr = op - ostart; + { + size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; + if (hintInSize==0) hintInSize = zbc->blockSize; + return hintInSize; + } +} + +size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, + void* dst, size_t* maxDstSizePtr, + const void* src, size_t* srcSizePtr) +{ return ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, src, srcSizePtr, 0); } + + + +/* *** Finalize *** */ + +size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) +{ + size_t srcSize = 0; + ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, NULL, &srcSize, 1); + return zbc->outBuffContentSize - zbc->outBuffFlushedSize; +} + + +size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + BYTE* const oend = ostart + *maxDstSizePtr; + size_t outSize = *maxDstSizePtr; + size_t epilogueSize, remaining; + ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */ + op += outSize; + epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */ + zbc->outBuffContentSize += epilogueSize; + outSize = oend-op; + zbc->stage = ZBUFFcs_flush; + remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */ + op += outSize; + if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */ + *maxDstSizePtr = op-ostart; /* tells how many bytes were written */ + return remaining; +} + + + +/** ************************************************ +* Streaming decompression +* +* A ZBUFF_DCtx object is required to track streaming operation. +* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. +* Use ZBUFF_decompressInit() to start a new decompression operation. +* ZBUFF_DCtx objects can be reused multiple times. +* +* Use ZBUFF_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *maxDstSizePtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. +* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. +* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* or 0 when a frame is completely decoded +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) +* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. +* input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* **************************************************/ + +typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader, + ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; + +/* *** Resource management *** */ + +#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */ +struct ZBUFF_DCtx_s { + ZSTD_DCtx* zc; + ZSTD_parameters params; + char* inBuff; + size_t inBuffSize; + size_t inPos; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t hPos; + ZBUFF_dStage stage; + unsigned char headerBuffer[ZSTD_frameHeaderSize_max]; +}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ + + +ZBUFF_DCtx* ZBUFF_createDCtx(void) +{ + ZBUFF_DCtx* zbc = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx)); + memset(zbc, 0, sizeof(*zbc)); + zbc->zc = ZSTD_createDCtx(); + zbc->stage = ZBUFFds_init; + return zbc; +} + +size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc) +{ + if (zbc==NULL) return 0; /* support free on null */ + ZSTD_freeDCtx(zbc->zc); + free(zbc); + return 0; +} + + + +/* *** Initialization *** */ + +size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) +{ + zbc->stage = ZBUFFds_readHeader; + zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0; + return ZSTD_resetDCtx(zbc->zc); +} + + + +/* *** Decompression *** */ + +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr) +{ + const char* const istart = (const char*)src; + const char* ip = istart; + const char* const iend = istart + *srcSizePtr; + char* const ostart = (char*)dst; + char* op = ostart; + char* const oend = ostart + *maxDstSizePtr; + U32 notDone = 1; + + while (notDone) + { + switch(zbc->stage) + { + + case ZBUFFds_init : + return ERROR(init_missing); + + case ZBUFFds_readHeader : + /* read header from src */ + { + size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr); + if (ZSTD_isError(headerSize)) return headerSize; + if (headerSize) + { + /* not enough input to decode header : tell how many bytes would be necessary */ + memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr); + zbc->hPos += *srcSizePtr; + *maxDstSizePtr = 0; + zbc->stage = ZBUFFds_loadHeader; + return headerSize - zbc->hPos; + } + zbc->stage = ZBUFFds_decodeHeader; + break; + } + + case ZBUFFds_loadHeader: + /* complete header from src */ + { + size_t headerSize = ZBUFF_limitCopy( + zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos, + src, *srcSizePtr); + zbc->hPos += headerSize; + ip += headerSize; + headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos); + if (ZSTD_isError(headerSize)) return headerSize; + if (headerSize) + { + /* not enough input to decode header : tell how many bytes would be necessary */ + *maxDstSizePtr = 0; + return headerSize - zbc->hPos; + } + // zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */ + } + + case ZBUFFds_decodeHeader: + /* apply header to create / resize buffers */ + { + size_t neededOutSize = (size_t)1 << zbc->params.windowLog; + size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ + if (zbc->inBuffSize < neededInSize) + { + free(zbc->inBuff); + zbc->inBuffSize = neededInSize; + zbc->inBuff = (char*)malloc(neededInSize); + if (zbc->inBuff == NULL) return ERROR(memory_allocation); + } + if (zbc->outBuffSize < neededOutSize) + { + free(zbc->outBuff); + zbc->outBuffSize = neededOutSize; + zbc->outBuff = (char*)malloc(neededOutSize); + if (zbc->outBuff == NULL) return ERROR(memory_allocation); + } + } + memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos); + zbc->inPos = zbc->hPos; + zbc->hPos = 0; + zbc->stage = ZBUFFds_load; + break; /* useless : stage follows */ + + case ZBUFFds_read: + { + size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); + if (neededInSize==0) /* end of frame */ + { + zbc->stage = ZBUFFds_init; + notDone = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) + { + /* directly decode from src */ + size_t decodedSize = ZSTD_decompressContinue(zbc->zc, + zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart, + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize) break; /* this was just a header */ + zbc->outEnd = zbc->outStart + decodedSize; + zbc->stage = ZBUFFds_flush; + break; + } + if (ip==iend) { notDone = 0; break; } /* no more input */ + zbc->stage = ZBUFFds_load; + } + + case ZBUFFds_load: + { + size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); + size_t toLoad = neededInSize - zbc->inPos; + size_t loadedSize; + if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, zbc->inBuffSize - zbc->inPos, ip, iend-ip); + ip += loadedSize; + zbc->inPos += loadedSize; + if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ + { + size_t decodedSize = ZSTD_decompressContinue(zbc->zc, + zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart, + zbc->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zbc->inPos = 0; /* input is consumed */ + if (!decodedSize) { zbc->stage = ZBUFFds_read; break; } /* this was just a header */ + zbc->outEnd = zbc->outStart + decodedSize; + zbc->stage = ZBUFFds_flush; + // break; /* ZBUFFds_flush follows */ + } + } + case ZBUFFds_flush: + { + size_t toFlushSize = zbc->outEnd - zbc->outStart; + size_t flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize); + op += flushedSize; + zbc->outStart += flushedSize; + if (flushedSize == toFlushSize) + { + zbc->stage = ZBUFFds_read; + if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize) + zbc->outStart = zbc->outEnd = 0; + } + break; + } + } + } + + *srcSizePtr = ip-istart; + *maxDstSizePtr = op-ostart; + + return ZSTD_nextSrcSizeToDecompress(zbc->zc) - zbc->inPos; +} + + + + + + + + +/* ************************************* +* Tool functions +***************************************/ +unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } +const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + +size_t ZBUFF_recommendedCInSize() { return BLOCKSIZE; } +size_t ZBUFF_recommendedCOutSize() { return ZSTD_compressBound(BLOCKSIZE) + 6; } +size_t ZBUFF_recommendedDInSize() { return BLOCKSIZE + 3; } +size_t ZBUFF_recommendedDOutSize() { return BLOCKSIZE; } diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h new file mode 100644 index 000000000..f79982a13 --- /dev/null +++ b/lib/zstd_buffered.h @@ -0,0 +1,146 @@ +/* + Buffered version of Zstd compression library + Copyright (C) 2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd source repository : https://github.com/Cyan4973/zstd + - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#ifndef ZSTD_BUFFERED_H +#define ZSTD_BUFFERED_H + +/* The objects defined into this file should be considered experimental. + * They are not labelled stable, as their prototype may change in the future. + * You can use them for tests, provide feedback, or if you can endure risk of future changes. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************* +* Includes +***************************************/ +#include /* size_t */ + + +/* ************************************* +* Streaming functions +***************************************/ +typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; +ZBUFF_CCtx* ZBUFF_createCCtx(void); +size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); + +size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); +size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); +size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); +size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); + +/** ************************************************ +* Streaming compression +* +* A ZBUFF_CCtx object is required to track streaming operation. +* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. +* Use ZBUFF_compressInit() to start a new compression operation. +* ZBUFF_CCtx objects can be reused multiple times. +* +* Use ZBUFF_compressContinue() repetitively to consume input stream. +* *srcSizePtr and *maxDstSizePtr can be any size. +* The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data. +* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or move dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. +* Note that it will not output more than *maxDstSizePtr. +* Therefore, some content might still be left into its internal buffer if dst buffer is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressEnd() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small. +* In which case, call again ZBUFF_compressFlush() to complete the flush. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedCInSize / ZBUFF_recommendedCOutSize +* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, it improves latency to use this value. +* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering. +* By using both, you ensure that input will be entirely consumed, and output will always contain the result. +* **************************************************/ + + +typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; +ZBUFF_DCtx* ZBUFF_createDCtx(void); +size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); + +size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); + +/** ************************************************ +* Streaming decompression +* +* A ZBUFF_DCtx object is required to track streaming operation. +* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. +* Use ZBUFF_decompressInit() to start a new decompression operation. +* ZBUFF_DCtx objects can be reused multiple times. +* +* Use ZBUFF_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *maxDstSizePtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. +* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. +* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* or 0 when a frame is completely decoded +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize +* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. +* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* **************************************************/ + + +/* ************************************* +* Tool functions +***************************************/ +unsigned ZBUFF_isError(size_t errorCode); +const char* ZBUFF_getErrorName(size_t errorCode); + +/** The below functions provide recommended buffer sizes for Compression or Decompression operations. +* These sizes are not compulsory, they just tend to offer better latency */ +size_t ZBUFF_recommendedCInSize(void); +size_t ZBUFF_recommendedCOutSize(void); +size_t ZBUFF_recommendedDInSize(void); +size_t ZBUFF_recommendedDOutSize(void); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_BUFFERED_H */ diff --git a/lib/zstd_buffered_static.h b/lib/zstd_buffered_static.h new file mode 100644 index 000000000..cba1d6fca --- /dev/null +++ b/lib/zstd_buffered_static.h @@ -0,0 +1,62 @@ +/* + zstd - buffered version of compression library + experimental complementary API, for static linking only + Copyright (C) 2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd source repository : https://github.com/Cyan4973/zstd + - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#ifndef ZSTD_BUFFERED_STATIC_H +#define ZSTD_BUFFERED_STATIC_H + +/* The objects defined into this file should be considered experimental. + * They are not labelled stable, as their prototype may change in the future. + * You can use them for tests, provide feedback, or if you can endure risk of future changes. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************* +* Includes +***************************************/ +#include "zstd_static.h" +#include "zstd_buffered.h" + + +/* ************************************* +* Advanced Streaming functions +***************************************/ +size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_BUFFERED_STATIC_H */ diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 0c9b0e873..21865d337 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -138,7 +138,7 @@ static unsigned ZSTD_highbit(U32 val); /** ZSTD_validateParams correct params value to remain within authorized range optimize for srcSize if srcSize > 0 */ -void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint) +void ZSTD_validateParams(ZSTD_parameters* params) { const U32 btPlus = (params->strategy == ZSTD_btlazy2); @@ -147,12 +147,13 @@ void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint) if (params->windowLog < ZSTD_WINDOWLOG_MIN) params->windowLog = ZSTD_WINDOWLOG_MIN; /* correct params, to use less memory */ - if ((srcSizeHint > 0) && (srcSizeHint < (1<srcSize > 0) && (params->srcSize < (1<srcSize)-1) + 1; if (params->windowLog > srcLog) params->windowLog = srcLog; } + if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus; /* <= ZSTD_CONTENTLOG_MAX */ if (params->contentLog < ZSTD_CONTENTLOG_MIN) params->contentLog = ZSTD_CONTENTLOG_MIN; if (params->hashLog > ZSTD_HASHLOG_MAX) params->hashLog = ZSTD_HASHLOG_MAX; @@ -166,11 +167,8 @@ void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint) static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, - ZSTD_parameters params, - U64 srcSizeHint) + ZSTD_parameters params) { - ZSTD_validateParams(¶ms, srcSizeHint); - /* reserve table memory */ { const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog; @@ -207,10 +205,12 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, } +/** ZSTD_reduceIndex +* rescale indexes to avoid future overflow (indexes are U32) */ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) { - const U32 contentLog = zc->params.strategy == ZSTD_fast ? 1 : zc->params.contentLog; + const U32 contentLog = (zc->params.strategy == ZSTD_fast) ? 1 : zc->params.contentLog; const U32 tableSpaceU32 = (1 << contentLog) + (1 << zc->params.hashLog); U32* table32 = zc->hashTable; U32 index; @@ -2038,30 +2038,51 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, } +/** ZSTD_compressBegin_advanced +* Write frame header, according to params +* @return : nb of bytes written */ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, - const ZSTD_parameters params, - const U64 srcSizeHint) + ZSTD_parameters params) { size_t errorCode; - if (maxDstSize < 4) return ERROR(dstSize_tooSmall); - errorCode = ZSTD_resetCCtx_advanced(ctx, params, srcSizeHint); + + ZSTD_validateParams(¶ms); + + if (maxDstSize < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); + errorCode = ZSTD_resetCCtx_advanced(ctx, params); if (ZSTD_isError(errorCode)) return errorCode; - MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */ - return 4; + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); /* Write Header */ + ((BYTE*)dst)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN); + return ZSTD_frameHeaderSize_min; } -size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint) +/** ZSTD_getParams +* return ZSTD_parameters structure for a selected compression level and srcSize. +* srcSizeHint value is optional, select 0 if not known */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint) { - int tableID = ((srcSizeHint-1) > 128 KB); /* intentional underflow for srcSizeHint == 0 */ + ZSTD_parameters result; + int tableID = ((srcSizeHint-1) <= 128 KB); /* intentional underflow for srcSizeHint == 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; - return ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_defaultParameters[tableID][compressionLevel], srcSizeHint); + result = ZSTD_defaultParameters[tableID][compressionLevel]; + result.srcSize = srcSizeHint; + return result; +} + + +size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint) +{ + return ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_getParams(compressionLevel, srcSizeHint)); } +/** ZSTD_compressEnd +* Write frame epilogue +* @return : nb of bytes written into dst (or an error code) */ size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize) { BYTE* op = (BYTE*)dst; @@ -2079,16 +2100,16 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize) } size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - ZSTD_parameters params) + void* dst, size_t maxDstSize, + const void* src, size_t srcSize, + ZSTD_parameters params) { BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; size_t oSize; /* Header */ - oSize = ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize); + oSize = ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, params); if(ZSTD_isError(oSize)) return oSize; op += oSize; maxDstSize -= oSize; @@ -2110,10 +2131,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) { - const int tableID = (srcSize > 128 KB); - if (compressionLevel < 1) compressionLevel = 1; - if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; - return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_defaultParameters[tableID][compressionLevel]); + return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_getParams(compressionLevel, srcSize)); } size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 430dc7aca..54429628d 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -120,6 +120,9 @@ const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } /* ************************************************************* * Context management ***************************************************************/ +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage; + struct ZSTD_DCtx_s { U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; @@ -130,18 +133,21 @@ struct ZSTD_DCtx_s void* vBase; void* dictEnd; size_t expected; + size_t headerSize; + ZSTD_parameters params; blockType_t bType; - U32 phase; + ZSTD_dStage stage; const BYTE* litPtr; size_t litBufSize; size_t litSize; BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */]; + BYTE headerBuffer[ZSTD_frameHeaderSize_max]; }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */ size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx) { - dctx->expected = ZSTD_frameHeaderSize; - dctx->phase = 0; + dctx->expected = ZSTD_frameHeaderSize_min; + dctx->stage = ZSTDds_getFrameHeaderSize; dctx->previousDstEnd = NULL; dctx->base = NULL; dctx->vBase = NULL; @@ -167,6 +173,45 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) /* ************************************************************* * Decompression section ***************************************************************/ +/** ZSTD_decodeFrameHeader_Part1 +* decode the 1st part of the Frame Header, which tells Frame Header size. +* srcSize must be == ZSTD_frameHeaderSize_min +* @return : the full size of the Frame Header */ +static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_t srcSize) +{ + U32 magicNumber; + if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); + magicNumber = MEM_readLE32(src); + if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + zc->headerSize = ZSTD_frameHeaderSize_min; + return zc->headerSize; +} + +/** ZSTD_decodeFrameHeader_Part2 +* decode the full Frame Header +* srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1 +* @return : 0, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + if (srcSize != zc->headerSize) return ERROR(srcSize_wrong); + memset(&(zc->params), 0, sizeof(zc->params)); + zc->params.windowLog = ip[4] + ZSTD_WINDOWLOG_ABSOLUTEMIN; + return 0; +} + + +size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize) +{ + U32 magicNumber; + if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max; + magicNumber = MEM_readLE32(src); + if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + memset(params, 0, sizeof(*params)); + params->windowLog = ((const BYTE*)src)[4] + ZSTD_WINDOWLOG_ABSOLUTEMIN; + return 0; +} + size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) { @@ -655,7 +700,6 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const v BYTE* op = ostart; BYTE* const oend = ostart + maxDstSize; size_t remainingSize = srcSize; - U32 magicNumber; blockProperties_t blockProperties; @@ -663,14 +707,23 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const v ctx->base = ctx->vBase = ctx->dictEnd = dst; /* Frame Header */ - if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); - magicNumber = MEM_readLE32(src); + { + size_t frameHeaderSize; + if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - if (ZSTD_isLegacy(magicNumber)) - return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber); + { + const U32 magicNumber = MEM_readLE32(src); + if (ZSTD_isLegacy(magicNumber)) + return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber); + } #endif - if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); - ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; + frameHeaderSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + frameHeaderSize = ZSTD_decodeFrameHeader_Part2(ctx, src, frameHeaderSize); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + } /* Loop on each block */ while (1) @@ -722,7 +775,6 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src /* ****************************** * Streaming Decompression API ********************************/ - size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; @@ -732,7 +784,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con { /* Sanity check */ if (srcSize != ctx->expected) return ERROR(srcSize_wrong); - if (dst != ctx->previousDstEnd) /* not contiguous */ + if (dst != ctx->previousDstEnd) /* not contiguous */ { ctx->dictEnd = ctx->previousDstEnd; if ((dst > ctx->base) && (dst < ctx->previousDstEnd)) /* rolling buffer : new segment right into tracked memory */ @@ -741,65 +793,85 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con ctx->base = dst; } - /* Decompress : frame header */ - if (ctx->phase == 0) - { - /* Check frame magic header */ - U32 magicNumber = MEM_readLE32(src); - if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); - ctx->phase = 1; - ctx->expected = ZSTD_blockHeaderSize; - return 0; - } - - /* Decompress : block header */ - if (ctx->phase == 1) + /* Decompress : frame header; part 1 */ + switch (ctx->stage) { - blockProperties_t bp; - size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); - if (ZSTD_isError(blockSize)) return blockSize; - if (bp.blockType == bt_end) + case ZSTDds_getFrameHeaderSize : { - ctx->expected = 0; - ctx->phase = 0; + /* get frame header size */ + if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min); + if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize; + memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + if (ctx->headerSize > ZSTD_frameHeaderSize_min) + { + ctx->expected = ctx->headerSize - ZSTD_frameHeaderSize_min; + ctx->stage = ZSTDds_decodeFrameHeader; + return 0; + } + ctx->expected = 0; /* not necessary to copy more */ } - else + case ZSTDds_decodeFrameHeader: { - ctx->expected = blockSize; - ctx->bType = bp.blockType; - ctx->phase = 2; + /* get frame header */ + size_t result; + memcpy(ctx->headerBuffer + ZSTD_frameHeaderSize_min, src, ctx->expected); + result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize); + if (ZSTD_isError(result)) return result; + ctx->expected = ZSTD_blockHeaderSize; + ctx->stage = ZSTDds_decodeBlockHeader; + return 0; } + case ZSTDds_decodeBlockHeader: + { + /* Decode block header */ + blockProperties_t bp; + size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(blockSize)) return blockSize; + if (bp.blockType == bt_end) + { + ctx->expected = 0; + ctx->stage = ZSTDds_getFrameHeaderSize; + } + else + { + ctx->expected = blockSize; + ctx->bType = bp.blockType; + ctx->stage = ZSTDds_decompressBlock; + } - ctx->previousDstEnd = dst; - return 0; - } - - /* Decompress : block content */ - { - size_t rSize; - switch(ctx->bType) + ctx->previousDstEnd = dst; + return 0; + } + case 3: { - case bt_compressed: - rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize); - break; - case bt_raw : - rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize); - break; - case bt_rle : - return ERROR(GENERIC); /* not yet handled */ - break; - case bt_end : /* should never happen (filtered at phase 1) */ - rSize = 0; - break; - default: - return ERROR(GENERIC); + /* Decompress : block content */ + size_t rSize; + switch(ctx->bType) + { + case bt_compressed: + rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize); + break; + case bt_raw : + rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize); + break; + case bt_rle : + return ERROR(GENERIC); /* not yet handled */ + break; + case bt_end : /* should never happen (filtered at phase 1) */ + rSize = 0; + break; + default: + return ERROR(GENERIC); + } + ctx->stage = ZSTDds_decodeBlockHeader; + ctx->expected = ZSTD_blockHeaderSize; + ctx->previousDstEnd = (char*)dst + rSize; + return rSize; } - ctx->phase = 1; - ctx->expected = ZSTD_blockHeaderSize; - ctx->previousDstEnd = (char*)dst + rSize; - return rSize; + default: + return ERROR(GENERIC); /* impossible */ } - } diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 7236cf2ab..bddfc9293 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -54,6 +54,8 @@ extern "C" { /* ************************************* * Common constants ***************************************/ +#define ZSTD_MAGICNUMBER 0xFD2FB524 /* v0.4 */ + #define KB *(1 <<10) #define MB *(1 <<20) #define GB *(1U<<30) @@ -61,7 +63,8 @@ extern "C" { #define BLOCKSIZE (128 KB) /* define, for static allocation */ static const size_t ZSTD_blockHeaderSize = 3; -static const size_t ZSTD_frameHeaderSize = 4; +static const size_t ZSTD_frameHeaderSize_min = 5; +#define ZSTD_frameHeaderSize_max 5 /* define, for static allocation */ #define BIT7 128 #define BIT6 64 diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 8c56c13e4..ed06d69dd 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -52,16 +52,29 @@ extern "C" { /* ************************************* * Types ***************************************/ +#define ZSTD_WINDOWLOG_MAX 26 +#define ZSTD_WINDOWLOG_MIN 18 +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11 +#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1) +#define ZSTD_CONTENTLOG_MIN 4 +#define ZSTD_HASHLOG_MAX 28 +#define ZSTD_HASHLOG_MIN 4 +#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_SEARCHLENGTH_MAX 7 +#define ZSTD_SEARCHLENGTH_MIN 4 + /** from faster to stronger */ typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy; typedef struct { - U32 windowLog; /* largest match distance : impact decompression buffer size */ + U64 srcSize; /* optional : tells how much bytes are present in the frame. Use 0 if not known. */ + U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */ U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */ - U32 hashLog; /* dispatch table : larger == more memory, faster*/ - U32 searchLog; /* nb of searches : larger == more compression, slower*/ - U32 searchLength; /* size of matches : larger == faster decompression */ + U32 hashLog; /* dispatch table : larger == more memory, faster */ + U32 searchLog; /* nb of searches : larger == more compression, slower */ + U32 searchLength; /* size of matches : larger == faster decompression, sometimes less compression */ ZSTD_strategy strategy; } ZSTD_parameters; @@ -69,23 +82,29 @@ typedef struct /* ************************************* * Advanced function ***************************************/ +/** ZSTD_getParams +* return ZSTD_parameters structure for a selected compression level and srcSize. +* srcSizeHint value is optional, select 0 if not known */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint); + +/** ZSTD_validateParams +* correct params value to remain within authorized range */ +void ZSTD_validateParams(ZSTD_parameters* params); + /** ZSTD_compress_advanced * Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - ZSTD_parameters params); - -/** ZSTD_validateParams - correct params value to remain within authorized range - srcSizeHint value is optional, select 0 if not known */ -void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint); + void* dst, size_t maxDstSize, + const void* src, size_t srcSize, + ZSTD_parameters params); -/* ************************************* -* Streaming functions -***************************************/ +/* ************************************** +* Streaming functions (bufferless mode) +****************************************/ size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint); +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params); + size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); @@ -95,86 +114,90 @@ ZSTD_DCtx* ZSTD_createDCtx(void); size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx); size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); +size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/* - Use above functions alternatively. +/** + Streaming decompression, bufferless mode + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. Use ZSTD_resetDCtx() to return to fresh status. + + First operation is to retrieve frame parameters, using ZSTD_getFrameParams(). + This function doesn't consume its input. It needs enough input data to properly decode the frame header. + The objective is to retrieve *params.windowlog, to know how much memory is required during decoding. + Result : 0 if successfull, it means the ZSTD_parameters structure has been filled. + >0 : means there is not enough data into src. Provides the expected size to successfully decode header. + errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header) + + Then it's possible to start decompression. + Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. + ZSTD_decompressContinue() will use previous data blocks during decompress. + They should be located contiguously prior to current block. Alternatively, a round buffer is possible. + Just make sure that the combined of current and accessible past blocks is a minimum of (1 << windowlog). + + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst'. It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTD_magicNumber 0xFD2FB523 /* v0.3 (current)*/ + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. +*/ /* ************************************* * Pre-defined compression levels ***************************************/ #define ZSTD_MAX_CLEVEL 20 -#define ZSTD_WINDOWLOG_MAX 26 -#define ZSTD_WINDOWLOG_MIN 18 -#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1) -#define ZSTD_CONTENTLOG_MIN 4 -#define ZSTD_HASHLOG_MAX 28 -#define ZSTD_HASHLOG_MIN 4 -#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -#define ZSTD_SEARCHLENGTH_MAX 7 -#define ZSTD_SEARCHLENGTH_MIN 4 - static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = { -{ /* for <= 128 KB */ - /* W, C, H, S, L, strat */ - { 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ - { 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */ - { 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */ - { 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */ - { 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */ - { 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */ - { 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */ - { 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */ - { 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */ - { 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */ - { 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */ - { 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */ - { 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */ - { 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */ - { 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */ - { 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */ - { 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */ - { 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */ - { 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */ - { 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */ - { 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */ +{ /* "default" */ + /* W, C, H, S, L, strat */ + { 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ + { 0, 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */ + { 0, 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */ + { 0, 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */ + { 0, 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */ + { 0, 20, 14, 18, 3, 5, ZSTD_greedy }, /* level 5 */ + { 0, 20, 18, 19, 3, 5, ZSTD_greedy }, /* level 6 */ + { 0, 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */ + { 0, 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */ + { 0, 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */ + { 0, 21, 19, 21, 4, 5, ZSTD_lazy2 }, /* level 10 */ + { 0, 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */ + { 0, 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */ + { 0, 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */ + { 0, 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */ + { 0, 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */ + { 0, 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */ + { 0, 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */ + { 0, 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */ + { 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */ + { 0, 25, 26, 25, 6, 5, ZSTD_btlazy2 }, /* level 20 */ }, -{ /* for > 128 KB */ +{ /* for srcSize <= 128 KB */ /* W, C, H, S, L, strat */ - { 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ - { 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */ - { 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */ - { 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */ - { 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */ - { 20, 14, 18, 3, 5, ZSTD_greedy }, /* level 5 */ - { 20, 18, 19, 3, 5, ZSTD_greedy }, /* level 6 */ - { 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */ - { 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */ - { 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */ - { 21, 19, 21, 4, 5, ZSTD_lazy2 }, /* level 10 */ - { 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */ - { 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */ - { 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */ - { 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */ - { 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */ - { 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */ - { 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */ - { 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */ - { 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */ - { 25, 26, 25, 6, 5, ZSTD_btlazy2 }, /* level 20 */ -} + { 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ + { 0, 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */ + { 0, 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */ + { 0, 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */ + { 0, 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */ + { 0, 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */ + { 0, 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */ + { 0, 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */ + { 0, 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */ + { 0, 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */ + { 0, 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */ + { 0, 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */ + { 0, 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */ + { 0, 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */ + { 0, 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */ + { 0, 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */ + { 0, 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */ + { 0, 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */ + { 0, 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */ + { 0, 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */ + { 0, 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */ +}, }; diff --git a/programs/Makefile b/programs/Makefile index eddd32583..697bfe40a 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -58,12 +58,12 @@ default: zstd all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen -zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \ +zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \ $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c $(CC) $(FLAGS) $^ -o $@$(EXT) -zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \ +zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \ $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) diff --git a/programs/fileio.c b/programs/fileio.c index c599c476a..267ddb562 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -49,8 +49,6 @@ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - #define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ #define _POSIX_SOURCE 1 /* enable fileno() within on unix */ @@ -67,10 +65,11 @@ #include /* stat64 */ #include "mem.h" #include "fileio.h" -#include "zstd_static.h" +#include "zstd_static.h" /* ZSTD_magicNumber */ +#include "zstd_buffered_static.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) -# include "zstd_legacy.h" /* legacy */ +# include "zstd_legacy.h" /* legacy */ # include "fileio_legacy.h" /* legacy */ #endif @@ -82,7 +81,7 @@ # include /* _O_BINARY */ # include /* _setmode, _isatty */ # ifdef __MINGW32__ - /* int _fileno(FILE *stream); // seems no longer useful // MINGW somehow forgets to include this windows declaration into */ + // int _fileno(FILE *stream); /* seems no longer useful /* MINGW somehow forgets to include this windows declaration into */ # endif # define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; } # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) @@ -114,8 +113,8 @@ #define BIT6 0x40 #define BIT7 0x80 -//static const unsigned FIO_maxBlockSizeID = 0xB; /* => 2MB block */ -static const unsigned FIO_blockHeaderSize = 3; +#define BLOCKSIZE (128 KB) +#define ROLLBUFFERSIZE (BLOCKSIZE*8*64) #define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */ #define FSE_CHECKSUM_SEED 0 @@ -241,36 +240,25 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* U64 filesize = 0; U64 compressedfilesize = 0; BYTE* inBuff; - BYTE* inSlot; - BYTE* inEnd; BYTE* outBuff; - size_t blockSize = 128 KB; - size_t inBuffSize = 4 * blockSize; - size_t outBuffSize = ZSTD_compressBound(blockSize); + size_t inBuffSize = ZBUFF_recommendedCInSize(); + size_t outBuffSize = ZBUFF_recommendedCOutSize(); FILE* finput; FILE* foutput; - size_t sizeCheck, cSize; - ZSTD_CCtx* ctx; - - /* init */ - FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); - filesize = FIO_getFileSize(input_filename); + size_t sizeCheck, errorCode; + ZBUFF_CCtx* ctx; /* Allocate Memory */ - ctx = ZSTD_createCCtx(); + ctx = ZBUFF_createCCtx(); inBuff = (BYTE*)malloc(inBuffSize); outBuff = (BYTE*)malloc(outBuffSize); if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory"); - inSlot = inBuff; - inEnd = inBuff + inBuffSize; - - /* Write Frame Header */ - cSize = ZSTD_compressBegin(ctx, outBuff, outBuffSize, cLevel, filesize); - if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header"); - sizeCheck = fwrite(outBuff, 1, cSize, foutput); - if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename); - compressedfilesize += cSize; + /* init */ + FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); + filesize = FIO_getFileSize(input_filename); + errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize)); + if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression"); filesize = 0; /* Main compression loop */ @@ -279,33 +267,41 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* size_t inSize; /* Fill input Buffer */ - if (inSlot + blockSize > inEnd) inSlot = inBuff; - inSize = fread(inSlot, (size_t)1, blockSize, finput); + inSize = fread(inBuff, (size_t)1, inBuffSize, finput); if (inSize==0) break; filesize += inSize; DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20)); - /* Compress Block */ - cSize = ZSTD_compressContinue(ctx, outBuff, outBuffSize, inSlot, inSize); - if (ZSTD_isError(cSize)) - EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize)); - - /* Write cBlock */ - sizeCheck = fwrite(outBuff, 1, cSize, foutput); - if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename); - compressedfilesize += cSize; - inSlot += inSize; + { + /* Compress (buffered streaming ensures appropriate formatting) */ + size_t usedInSize = inSize; + size_t cSize = outBuffSize; + size_t result = ZBUFF_compressContinue(ctx, outBuff, &cSize, inBuff, &usedInSize); + if (ZBUFF_isError(result)) + EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result)); + if (inSize != usedInSize) + /* inBuff should be entirely consumed since buffer sizes are recommended ones */ + EXM_THROW(24, "Compression error : input block not fully consumed"); + + /* Write cBlock */ + sizeCheck = fwrite(outBuff, 1, cSize, foutput); + if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename); + compressedfilesize += cSize; + } DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100); } /* End of Frame */ - cSize = ZSTD_compressEnd(ctx, outBuff, outBuffSize); - if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end"); + { + size_t cSize = outBuffSize; + size_t result = ZBUFF_compressEnd(ctx, outBuff, &cSize); + if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end"); - sizeCheck = fwrite(outBuff, 1, cSize, foutput); - if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename); - compressedfilesize += cSize; + sizeCheck = fwrite(outBuff, 1, cSize, foutput); + if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename); + compressedfilesize += cSize; + } /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); @@ -315,7 +311,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* /* clean */ free(inBuff); free(outBuff); - ZSTD_freeCCtx(ctx); + ZBUFF_freeCCtx(ctx); fclose(finput); if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename); @@ -324,124 +320,87 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, - BYTE* inBuff, size_t inBuffSize, + BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded, BYTE* outBuff, size_t outBuffSize, - ZSTD_DCtx* dctx) + ZBUFF_DCtx* dctx) { - BYTE* op = outBuff; - BYTE* const oend = outBuff + outBuffSize; - U64 filesize = 0; - size_t toRead; - size_t sizeCheck; - + U64 frameSize = 0; + size_t readSize=alreadyLoaded; /* Main decompression Loop */ - toRead = ZSTD_nextSrcSizeToDecompress(dctx); - while (toRead) + ZBUFF_decompressInit(dctx); + while (1) { - size_t readSize, decodedSize; + /* Decode */ + size_t sizeCheck; + size_t inSize=readSize, decodedSize=outBuffSize; + size_t inStart=0; + size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize); + if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); + if (toRead==0) break; /* end of Frame */ + readSize -= inSize; + inStart += inSize; + + /* Write block */ + sizeCheck = fwrite(outBuff, 1, decodedSize, foutput); + if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); + frameSize += decodedSize; + DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); + + if (readSize) continue; /* still some data left within inBuff */ /* Fill input buffer */ - if (toRead > inBuffSize) - EXM_THROW(34, "too large block"); + if (toRead > inBuffSize) EXM_THROW(34, "too large block"); readSize = fread(inBuff, 1, toRead, finput); - if (readSize != toRead) - EXM_THROW(35, "Read error"); - - /* Decode block */ - decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize); - if (ZSTD_isError(decodedSize)) EXM_THROW(36, "Decoding error : input corrupted"); - - if (decodedSize) /* not a header */ - { - /* Write block */ - sizeCheck = fwrite(op, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); - filesize += decodedSize; - op += decodedSize; - if (op==oend) op = outBuff; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) ); - } - - /* prepare for next Block */ - toRead = ZSTD_nextSrcSizeToDecompress(dctx); + if (readSize != toRead) EXM_THROW(35, "Read error"); } - return filesize; + return frameSize; } -#define MAXHEADERSIZE (FIO_FRAMEHEADERSIZE+3) unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename) { FILE* finput, *foutput; BYTE* inBuff=NULL; - size_t inBuffSize = 0; + size_t inBuffSize = ZBUFF_recommendedDInSize(); BYTE* outBuff=NULL; - size_t outBuffSize = 0; - U32 blockSize = 128 KB; - U32 wNbBlocks = 4; + size_t outBuffSize = ZBUFF_recommendedDOutSize(); U64 filesize = 0; - BYTE* header[MAXHEADERSIZE]; size_t toRead; size_t sizeCheck; /* Init */ - ZSTD_DCtx* dctx = ZSTD_createDCtx(); + ZBUFF_DCtx* dctx = ZBUFF_createDCtx(); FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); + /* Allocate Memory (if needed) */ + inBuff = (BYTE*)malloc(inBuffSize); + outBuff = (BYTE*)malloc(outBuffSize); + if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory"); + /* for each frame */ for ( ; ; ) { - /* check magic number -> version */ U32 magicNumber; - toRead = sizeof(ZSTD_magicNumber);; - sizeCheck = fread(header, (size_t)1, toRead, finput); - if (sizeCheck==0) break; /* no more input */ - if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header"); + toRead = 0; - magicNumber = MEM_readLE32(header); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + /* check magic number -> version */ + toRead = 4; + sizeCheck = fread(inBuff, (size_t)1, toRead, finput); + if (sizeCheck==0) break; /* no more input */ + if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header"); + magicNumber = MEM_readLE32(inBuff); if (ZSTD_isLegacy(magicNumber)) { filesize += FIO_decompressLegacyFrame(foutput, finput, magicNumber); continue; } #endif /* ZSTD_LEGACY_SUPPORT */ - if (magicNumber != ZSTD_magicNumber) EXM_THROW(32, "Error : unknown frame prefix"); - - /* prepare frame decompression, by completing header */ - ZSTD_resetDCtx(dctx); - toRead = ZSTD_nextSrcSizeToDecompress(dctx) - sizeof(ZSTD_magicNumber); - if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header"); - sizeCheck = fread(&header[sizeof(ZSTD_magicNumber)], 1, toRead, finput); - if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header"); - sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, sizeof(ZSTD_magicNumber)+toRead); // Decode frame header - if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header"); - - /* Here later : blockSize determination */ - - /* Allocate Memory (if needed) */ - { - size_t newInBuffSize = blockSize + FIO_blockHeaderSize; - size_t newOutBuffSize = wNbBlocks * blockSize; - if (newInBuffSize > inBuffSize) - { - free(inBuff); - inBuffSize = newInBuffSize; - inBuff = (BYTE*)malloc(inBuffSize); - } - if (newOutBuffSize > outBuffSize) - { - free(outBuff); - outBuffSize = newOutBuffSize; - outBuff = (BYTE*)malloc(outBuffSize); - } - } - if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory"); - filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx); + filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, toRead, outBuff, outBuffSize, dctx); } DISPLAYLEVEL(2, "\r%79s\r", ""); @@ -450,7 +409,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha /* clean */ free(inBuff); free(outBuff); - ZSTD_freeDCtx(dctx); + ZBUFF_freeDCtx(dctx); fclose(finput); if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename); @@ -458,74 +417,3 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha } -#if 0 -unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename) -{ - FILE* finput, *foutput; - BYTE* inBuff=NULL; - size_t inBuffSize = 0; - BYTE* outBuff=NULL; - size_t outBuffSize = 0; - U32 blockSize = 128 KB; - U32 wNbBlocks = 4; - U64 filesize = 0; - BYTE* header[MAXHEADERSIZE]; - ZSTD_Dctx* dctx; - size_t toRead; - size_t sizeCheck; - - - /* Init */ - FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); - dctx = ZSTD_createDCtx(); - - /* for each frame */ - for ( ; ; ) - { - /* check header */ - ZSTD_resetDCtx(dctx); - toRead = ZSTD_nextSrcSizeToDecompress(dctx); - if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header"); - sizeCheck = fread(header, (size_t)1, toRead, finput); - if (sizeCheck==0) break; /* no more input */ - if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header"); - sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header - if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header"); - - /* Here later : blockSize determination */ - - /* Allocate Memory (if needed) */ - { - size_t newInBuffSize = blockSize + FIO_blockHeaderSize; - size_t newOutBuffSize = wNbBlocks * blockSize; - if (newInBuffSize > inBuffSize) - { - free(inBuff); - inBuffSize = newInBuffSize; - inBuff = (BYTE*)malloc(inBuffSize); - } - if (newOutBuffSize > outBuffSize) - { - free(outBuff); - outBuffSize = newOutBuffSize; - outBuff = (BYTE*)malloc(outBuffSize); - } - } - if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory"); - - filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx); - } - - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Decoded %llu bytes \n", (long long unsigned)filesize); - - /* clean */ - free(inBuff); - free(outBuff); - ZSTD_freeDCtx(dctx); - fclose(finput); - if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename); - - return filesize; -} -#endif diff --git a/programs/fullbench.c b/programs/fullbench.c index 5ee5d4931..cb4822022 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -243,31 +243,6 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize); } -size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) -{ - U32 i; - size_t total = 0; - BYTE* data = (BYTE*)buff2; - - (void)dst; (void)dstSize; (void)src; - for (i=0; i < srcSize; i++) - { - U32 b = data[i]; - total += b; - if (b==0) total = 0; // 825 - //if (!b) total = 0; // 825 - //total = b ? total : 0; // 622 - //total &= -!b; // 622 - //total *= !!b; // 465 - } - return total; -} - -size_t local_decodeLiteralsForward(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) -{ - (void)src; (void)srcSize; - return FSE_decompress(dst, dstSize, buff2, g_cSize); -} @@ -300,12 +275,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) case 32: benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders"; break; - case 101: - benchFunction = local_conditionalNull; benchName = "conditionalNull"; - break; - case 102: - benchFunction = local_decodeLiteralsForward; benchName = "ZSTD_decodeLiteralsForward"; - break; default : return 0; } @@ -332,14 +301,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) { blockProperties_t bp; g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); - ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block type + ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); /* Get 1st block type */ if (bp.blockType != bt_compressed) { DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); goto _cleanOut; } - memcpy(buff2, dstBuff+7, g_cSize-7); - srcSize = srcSize > 128 KB ? 128 KB : srcSize; // relative to block + memcpy(buff2, dstBuff+8, g_cSize-8); + srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ break; } case 32: /* ZSTD_decodeSeqHeaders */ @@ -348,9 +317,9 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) const BYTE* ip = dstBuff; const BYTE* iend; size_t blockSize; - ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); - ip += 4; // Jump magic Number - blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block type + ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */ + ip += 5; /* Skip frame Header */ + blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ if (bp.blockType != bt_compressed) { DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); @@ -358,32 +327,16 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) } iend = ip + 3 + blockSize; /* End of first block */ ip += 3; /* skip block header */ - ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip); // jump literal sub block and its header + ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip); /* skip literal segment */ g_cSize = iend-ip; - memcpy(buff2, ip, g_cSize); // copy rest of block (starting with SeqHeader) - srcSize = srcSize > 128 KB ? 128 KB : srcSize; // speed relative to block + memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ + srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ break; } /* test functions */ + /* by convention, test functions can be added > 100 */ - case 101: /* conditionalNull */ - { - size_t i; - for (i=0; i 128 KB ? 128 KB : srcSize; // relative to block - break; - } default : ; } diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 47df78ab9..56fb621ad 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -46,7 +46,7 @@ #if defined(_MSC_VER) # define snprintf _snprintf /* snprintf unsupported by Visual <= 2012 */ #endif - + /************************************** * Includes @@ -125,8 +125,7 @@ static U32 g_rand = 1; static U32 g_singleRun = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static const ZSTD_parameters* g_seedParams = ZSTD_defaultParameters[0]; -static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_greedy }; +static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; void BMK_SetNbIterations(int nbLoops) { @@ -139,28 +138,6 @@ void BMK_SetNbIterations(int nbLoops) * Private functions *********************************************************/ -static unsigned BMK_highbit(U32 val) -{ -# if defined(_MSC_VER) /* Visual */ - unsigned long r; - _BitScanReverse(&r, val); - return (unsigned)r; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return 31 - __builtin_clz(val); -# else /* Software version */ - static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - int r; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; - return r; -# endif -} - #if defined(BMK_LEGACY_TIMER) static int BMK_GetMilliStart(void) @@ -655,7 +632,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, /* validate new conf */ { ZSTD_parameters saved = p; - ZSTD_validateParams(&p, g_blockSize ? g_blockSize : srcSize); + ZSTD_validateParams(&p); if (memcmp(&p, &saved, sizeof(p))) continue; /* p was invalid */ } @@ -707,12 +684,12 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) const char* rfName = "grillResults.txt"; FILE* f; const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - const U32 srcLog = BMK_highbit((U32)(blockSize-1))+1; if (g_singleRun) { BMK_result_t testResult; - ZSTD_validateParams(&g_params, blockSize); + g_params.srcSize = blockSize; + ZSTD_validateParams(&g_params); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params); DISPLAY("\n"); return; @@ -735,9 +712,10 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) params.searchLog = 1; params.searchLength = 7; params.strategy = ZSTD_fast; - ZSTD_validateParams(¶ms, blockSize); + params.srcSize = blockSize; + ZSTD_validateParams(¶ms); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); - g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4; + g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5; } /* establish speed objectives (relative to level 1) */ @@ -746,16 +724,10 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) /* populate initial solution */ { - const int tableID = (blockSize > 128 KB); const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL; - g_seedParams = ZSTD_defaultParameters[tableID]; for (i=1; i<=maxSeeds; i++) { - const U32 btPlus = (params.strategy == ZSTD_btlazy2); - params = g_seedParams[i]; - params.windowLog = MIN(srcLog, params.windowLog); - params.contentLog = MIN(params.windowLog+btPlus, params.contentLog); - params.searchLog = MIN(params.contentLog, params.searchLog); + params = ZSTD_getParams(i, blockSize); BMK_seed(winners, params, srcBuffer, srcSize, ctx); } } @@ -963,7 +935,7 @@ int main(int argc, char** argv) case 'S': g_singleRun = 1; argument++; - g_params = g_seedParams[2]; + g_params = ZSTD_getParams(2, g_blockSize); for ( ; ; ) { switch(*argument) @@ -1013,7 +985,7 @@ int main(int argc, char** argv) argument++; while ((*argument>= '0') && (*argument<='9')) cLevel *= 10, cLevel += *argument++ - '0'; - g_params = g_seedParams[cLevel]; + g_params = ZSTD_getParams(cLevel, g_blockSize); continue; } default : ;