PRGDIR = programs
ZSTDDIR = lib
-.PHONY: clean
+# Define nul output
+ifneq (,$(filter Windows%,$(OS)))
+VOID = nul
+else
+VOID = /dev/null
+endif
+
+.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
default: zstdprogram
$(MAKE) -C $(PRGDIR)
clean:
- $(MAKE) -C $(ZSTDDIR) $@
- $(MAKE) -C $(PRGDIR) $@
+ @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
+ @$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@echo Cleaning completed
v0.4.0
+Command line utility is now compatible with high compression levels
Removed zstdhc => merged into zstd
+Added : ZBUFF API (see zstd_buffered.h)
Rolling buffer support
v0.3.6
#define ERROR_LIST(ITEM) \
ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
- ITEM(PREFIX(mode_unsupported)) \
+ ITEM(PREFIX(mode_unsupported)) ITEM(PREFIX(init_missing))\
ITEM(PREFIX(memory_allocation)) \
ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \
--- /dev/null
+/*
+ Buffered version of Zstd compression library
+ Copyright (C) 2015, Yann Collet.
+
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - zstd source repository : https://github.com/Cyan4973/zstd
+ - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+/* The objects defined into this file should be considered experimental.
+ * They are not labelled stable, as their prototype may change in the future.
+ * You can use them for tests, provide feedback, or if you can endure risk of future changes.
+ */
+
+/* *************************************
+* Includes
+***************************************/
+#include <stdlib.h>
+#include "error.h"
+#include "zstd_static.h"
+#include "zstd_buffered_static.h"
+
+
+/** ************************************************
+* Streaming compression
+*
+* A ZBUFF_CCtx object is required to track streaming operation.
+* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
+* Use ZBUFF_compressInit() to start a new compression operation.
+* ZBUFF_CCtx objects can be reused multiple times.
+*
+* Use ZBUFF_compressContinue() repetitively to consume your input.
+* *srcSizePtr and *maxDstSizePtr can be any size.
+* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
+* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
+* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
+* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
+* Note that it will not output more than *maxDstSizePtr.
+* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
+* @return : nb of bytes still present into internal buffer (0 if it's empty)
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* ZBUFF_compressEnd() instructs to finish a frame.
+* It will perform a flush and write frame epilogue.
+* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small.
+* @return : nb of bytes still present into internal buffer (0 if it's empty)
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* Hint : recommended buffer sizes (not compulsory)
+* input : 128 KB block size is the internal unit, it improves latency to use this value.
+* output : ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block at best speed.
+* **************************************************/
+
+typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
+
+/* *** Ressources *** */
+struct ZBUFF_CCtx_s {
+ ZSTD_CCtx* zc;
+ char* inBuff;
+ size_t inBuffSize;
+ size_t inToCompress;
+ size_t inBuffPos;
+ size_t inBuffTarget;
+ size_t blockSize;
+ char* outBuff;
+ size_t outBuffSize;
+ size_t outBuffContentSize;
+ size_t outBuffFlushedSize;
+ ZBUFF_cStage stage;
+}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
+
+ZBUFF_CCtx* ZBUFF_createCCtx(void)
+{
+ ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx));
+ memset(zbc, 0, sizeof(*zbc));
+ zbc->zc = ZSTD_createCCtx();
+ return zbc;
+}
+
+size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
+{
+ if (zbc==NULL) return 0; /* support free on NULL */
+ ZSTD_freeCCtx(zbc->zc);
+ free(zbc);
+ return 0;
+}
+
+
+/* *** Initialization *** */
+
+#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) )
+#define BLOCKSIZE (128 * 1024) /* a bit too "magic", should come from reference */
+size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, ZSTD_parameters params)
+{
+ size_t neededInBuffSize;
+
+ ZSTD_validateParams(¶ms);
+ neededInBuffSize = (size_t)1 << params.windowLog;
+
+ /* allocate buffers */
+ if (zbc->inBuffSize < neededInBuffSize)
+ {
+ zbc->inBuffSize = neededInBuffSize;
+ free(zbc->inBuff); /* should not be necessary */
+ zbc->inBuff = (char*)malloc(neededInBuffSize);
+ if (zbc->inBuff == NULL) return ERROR(memory_allocation);
+ }
+ zbc->blockSize = MIN(BLOCKSIZE, zbc->inBuffSize);
+ if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1)
+ {
+ zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
+ free(zbc->outBuff); /* should not be necessary */
+ zbc->outBuff = (char*)malloc(zbc->outBuffSize);
+ if (zbc->outBuff == NULL) return ERROR(memory_allocation);
+ }
+
+ zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, zbc->outBuff, zbc->outBuffSize, params);
+ if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize;
+
+ zbc->inToCompress = 0;
+ zbc->inBuffPos = 0;
+ zbc->inBuffTarget = zbc->blockSize;
+ zbc->outBuffFlushedSize = 0;
+ zbc->stage = ZBUFFcs_flush; /* starts by flushing the header */
+ return 0; /* ready to go */
+}
+
+size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
+{
+ return ZBUFF_compressInit_advanced(zbc, ZSTD_getParams(compressionLevel, 0));
+}
+
+
+
+/* *** Compression *** */
+
+static size_t ZBUFF_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+ size_t length = MIN(maxDstSize, srcSize);
+ memcpy(dst, src, length);
+ return length;
+}
+
+static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
+ void* dst, size_t* maxDstSizePtr,
+ const void* src, size_t* srcSizePtr,
+ int flush) /* aggregate : wait for full block before compressing */
+{
+ U32 notDone = 1;
+ const char* const istart = (const char*)src;
+ const char* ip = istart;
+ const char* const iend = istart + *srcSizePtr;
+ char* const ostart = (char*)dst;
+ char* op = ostart;
+ char* const oend = ostart + *maxDstSizePtr;
+
+ while (notDone)
+ {
+ switch(zbc->stage)
+ {
+ case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
+
+ case ZBUFFcs_load:
+ /* complete inBuffer */
+ {
+ size_t toLoad = zbc->inBuffTarget - zbc->inBuffPos;
+ size_t loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
+ zbc->inBuffPos += loaded;
+ ip += loaded;
+ if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) )
+ { notDone = 0; break; } /* not enough input to get a full block : stop there, wait for more */
+ }
+ /* compress current block (note : this stage cannot be stopped in the middle) */
+ {
+ void* cDst;
+ size_t cSize;
+ size_t iSize = zbc->inBuffPos - zbc->inToCompress;
+ if ((size_t)(oend-op) > ZSTD_compressBound(iSize))
+ cDst = op; /* compress directly into output buffer (avoid flush stage) */
+ else
+ cDst = zbc->outBuff;
+ cSize = ZSTD_compressContinue(zbc->zc, cDst, oend-op, zbc->inBuff + zbc->inToCompress, iSize);
+ if (ZSTD_isError(cSize)) return cSize;
+ /* prepare next block */
+ zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
+ if (zbc->inBuffTarget > zbc->inBuffSize)
+ { zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; }
+ zbc->inToCompress = zbc->inBuffPos;
+ if (cDst == op) { op += cSize; break; } /* no need to flush */
+ zbc->outBuffContentSize = cSize;
+ zbc->outBuffFlushedSize = 0;
+ zbc->stage = ZBUFFcs_flush;
+ // break; /* flush stage follows */
+ }
+
+ case ZBUFFcs_flush:
+ /* flush into dst */
+ {
+ size_t toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
+ size_t flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
+ op += flushed;
+ zbc->outBuffFlushedSize += flushed;
+ if (toFlush!=flushed)
+ { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
+ zbc->outBuffContentSize = 0;
+ zbc->outBuffFlushedSize = 0;
+ zbc->stage = ZBUFFcs_load;
+ break;
+ }
+ }
+ }
+
+ *srcSizePtr = ip - istart;
+ *maxDstSizePtr = op - ostart;
+ {
+ size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
+ if (hintInSize==0) hintInSize = zbc->blockSize;
+ return hintInSize;
+ }
+}
+
+size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
+ void* dst, size_t* maxDstSizePtr,
+ const void* src, size_t* srcSizePtr)
+{ return ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, src, srcSizePtr, 0); }
+
+
+
+/* *** Finalize *** */
+
+size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
+{
+ size_t srcSize = 0;
+ ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, NULL, &srcSize, 1);
+ return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
+}
+
+
+size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* op = ostart;
+ BYTE* const oend = ostart + *maxDstSizePtr;
+ size_t outSize = *maxDstSizePtr;
+ size_t epilogueSize, remaining;
+ ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
+ op += outSize;
+ epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */
+ zbc->outBuffContentSize += epilogueSize;
+ outSize = oend-op;
+ zbc->stage = ZBUFFcs_flush;
+ remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */
+ op += outSize;
+ if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
+ *maxDstSizePtr = op-ostart; /* tells how many bytes were written */
+ return remaining;
+}
+
+
+
+/** ************************************************
+* Streaming decompression
+*
+* A ZBUFF_DCtx object is required to track streaming operation.
+* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
+* Use ZBUFF_decompressInit() to start a new decompression operation.
+* ZBUFF_DCtx objects can be reused multiple times.
+*
+* Use ZBUFF_decompressContinue() repetitively to consume your input.
+* *srcSizePtr and *maxDstSizePtr can be any size.
+* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
+* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
+* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
+* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+* or 0 when a frame is completely decoded
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* Hint : recommended buffer sizes (not compulsory)
+* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
+* input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
+* **************************************************/
+
+typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader,
+ ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
+
+/* *** Resource management *** */
+
+#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */
+struct ZBUFF_DCtx_s {
+ ZSTD_DCtx* zc;
+ ZSTD_parameters params;
+ char* inBuff;
+ size_t inBuffSize;
+ size_t inPos;
+ char* outBuff;
+ size_t outBuffSize;
+ size_t outStart;
+ size_t outEnd;
+ size_t hPos;
+ ZBUFF_dStage stage;
+ unsigned char headerBuffer[ZSTD_frameHeaderSize_max];
+}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
+
+
+ZBUFF_DCtx* ZBUFF_createDCtx(void)
+{
+ ZBUFF_DCtx* zbc = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx));
+ memset(zbc, 0, sizeof(*zbc));
+ zbc->zc = ZSTD_createDCtx();
+ zbc->stage = ZBUFFds_init;
+ return zbc;
+}
+
+size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
+{
+ if (zbc==NULL) return 0; /* support free on null */
+ ZSTD_freeDCtx(zbc->zc);
+ free(zbc);
+ return 0;
+}
+
+
+
+/* *** Initialization *** */
+
+size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
+{
+ zbc->stage = ZBUFFds_readHeader;
+ zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0;
+ return ZSTD_resetDCtx(zbc->zc);
+}
+
+
+
+/* *** Decompression *** */
+
+size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr)
+{
+ const char* const istart = (const char*)src;
+ const char* ip = istart;
+ const char* const iend = istart + *srcSizePtr;
+ char* const ostart = (char*)dst;
+ char* op = ostart;
+ char* const oend = ostart + *maxDstSizePtr;
+ U32 notDone = 1;
+
+ while (notDone)
+ {
+ switch(zbc->stage)
+ {
+
+ case ZBUFFds_init :
+ return ERROR(init_missing);
+
+ case ZBUFFds_readHeader :
+ /* read header from src */
+ {
+ size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr);
+ if (ZSTD_isError(headerSize)) return headerSize;
+ if (headerSize)
+ {
+ /* not enough input to decode header : tell how many bytes would be necessary */
+ memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr);
+ zbc->hPos += *srcSizePtr;
+ *maxDstSizePtr = 0;
+ zbc->stage = ZBUFFds_loadHeader;
+ return headerSize - zbc->hPos;
+ }
+ zbc->stage = ZBUFFds_decodeHeader;
+ break;
+ }
+
+ case ZBUFFds_loadHeader:
+ /* complete header from src */
+ {
+ size_t headerSize = ZBUFF_limitCopy(
+ zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos,
+ src, *srcSizePtr);
+ zbc->hPos += headerSize;
+ ip += headerSize;
+ headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos);
+ if (ZSTD_isError(headerSize)) return headerSize;
+ if (headerSize)
+ {
+ /* not enough input to decode header : tell how many bytes would be necessary */
+ *maxDstSizePtr = 0;
+ return headerSize - zbc->hPos;
+ }
+ // zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */
+ }
+
+ case ZBUFFds_decodeHeader:
+ /* apply header to create / resize buffers */
+ {
+ size_t neededOutSize = (size_t)1 << zbc->params.windowLog;
+ size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
+ if (zbc->inBuffSize < neededInSize)
+ {
+ free(zbc->inBuff);
+ zbc->inBuffSize = neededInSize;
+ zbc->inBuff = (char*)malloc(neededInSize);
+ if (zbc->inBuff == NULL) return ERROR(memory_allocation);
+ }
+ if (zbc->outBuffSize < neededOutSize)
+ {
+ free(zbc->outBuff);
+ zbc->outBuffSize = neededOutSize;
+ zbc->outBuff = (char*)malloc(neededOutSize);
+ if (zbc->outBuff == NULL) return ERROR(memory_allocation);
+ }
+ }
+ memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos);
+ zbc->inPos = zbc->hPos;
+ zbc->hPos = 0;
+ zbc->stage = ZBUFFds_load;
+ break; /* useless : stage follows */
+
+ case ZBUFFds_read:
+ {
+ size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
+ if (neededInSize==0) /* end of frame */
+ {
+ zbc->stage = ZBUFFds_init;
+ notDone = 0;
+ break;
+ }
+ if ((size_t)(iend-ip) >= neededInSize)
+ {
+ /* directly decode from src */
+ size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
+ zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
+ ip, neededInSize);
+ if (ZSTD_isError(decodedSize)) return decodedSize;
+ ip += neededInSize;
+ if (!decodedSize) break; /* this was just a header */
+ zbc->outEnd = zbc->outStart + decodedSize;
+ zbc->stage = ZBUFFds_flush;
+ break;
+ }
+ if (ip==iend) { notDone = 0; break; } /* no more input */
+ zbc->stage = ZBUFFds_load;
+ }
+
+ case ZBUFFds_load:
+ {
+ size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
+ size_t toLoad = neededInSize - zbc->inPos;
+ size_t loadedSize;
+ if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
+ loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, zbc->inBuffSize - zbc->inPos, ip, iend-ip);
+ ip += loadedSize;
+ zbc->inPos += loadedSize;
+ if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
+ {
+ size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
+ zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
+ zbc->inBuff, neededInSize);
+ if (ZSTD_isError(decodedSize)) return decodedSize;
+ zbc->inPos = 0; /* input is consumed */
+ if (!decodedSize) { zbc->stage = ZBUFFds_read; break; } /* this was just a header */
+ zbc->outEnd = zbc->outStart + decodedSize;
+ zbc->stage = ZBUFFds_flush;
+ // break; /* ZBUFFds_flush follows */
+ }
+ }
+ case ZBUFFds_flush:
+ {
+ size_t toFlushSize = zbc->outEnd - zbc->outStart;
+ size_t flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
+ op += flushedSize;
+ zbc->outStart += flushedSize;
+ if (flushedSize == toFlushSize)
+ {
+ zbc->stage = ZBUFFds_read;
+ if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize)
+ zbc->outStart = zbc->outEnd = 0;
+ }
+ break;
+ }
+ }
+ }
+
+ *srcSizePtr = ip-istart;
+ *maxDstSizePtr = op-ostart;
+
+ return ZSTD_nextSrcSizeToDecompress(zbc->zc) - zbc->inPos;
+}
+
+
+
+
+
+
+
+
+/* *************************************
+* Tool functions
+***************************************/
+unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
+const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
+
+size_t ZBUFF_recommendedCInSize() { return BLOCKSIZE; }
+size_t ZBUFF_recommendedCOutSize() { return ZSTD_compressBound(BLOCKSIZE) + 6; }
+size_t ZBUFF_recommendedDInSize() { return BLOCKSIZE + 3; }
+size_t ZBUFF_recommendedDOutSize() { return BLOCKSIZE; }
--- /dev/null
+/*
+ Buffered version of Zstd compression library
+ Copyright (C) 2015, Yann Collet.
+
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - zstd source repository : https://github.com/Cyan4973/zstd
+ - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#ifndef ZSTD_BUFFERED_H
+#define ZSTD_BUFFERED_H
+
+/* The objects defined into this file should be considered experimental.
+ * They are not labelled stable, as their prototype may change in the future.
+ * You can use them for tests, provide feedback, or if you can endure risk of future changes.
+ */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* *************************************
+* Includes
+***************************************/
+#include <stddef.h> /* size_t */
+
+
+/* *************************************
+* Streaming functions
+***************************************/
+typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
+ZBUFF_CCtx* ZBUFF_createCCtx(void);
+size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
+
+size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
+size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
+size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
+size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
+
+/** ************************************************
+* Streaming compression
+*
+* A ZBUFF_CCtx object is required to track streaming operation.
+* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
+* Use ZBUFF_compressInit() to start a new compression operation.
+* ZBUFF_CCtx objects can be reused multiple times.
+*
+* Use ZBUFF_compressContinue() repetitively to consume input stream.
+* *srcSizePtr and *maxDstSizePtr can be any size.
+* The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr.
+* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
+* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or move dst .
+* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
+* Note that it will not output more than *maxDstSizePtr.
+* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
+* @return : nb of bytes still present into internal buffer (0 if it's empty)
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* ZBUFF_compressEnd() instructs to finish a frame.
+* It will perform a flush and write frame epilogue.
+* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small.
+* In which case, call again ZBUFF_compressFlush() to complete the flush.
+* @return : nb of bytes still present into internal buffer (0 if it's empty)
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedCInSize / ZBUFF_recommendedCOutSize
+* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, it improves latency to use this value.
+* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
+* By using both, you ensure that input will be entirely consumed, and output will always contain the result.
+* **************************************************/
+
+
+typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
+ZBUFF_DCtx* ZBUFF_createDCtx(void);
+size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
+
+size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
+size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
+
+/** ************************************************
+* Streaming decompression
+*
+* A ZBUFF_DCtx object is required to track streaming operation.
+* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
+* Use ZBUFF_decompressInit() to start a new decompression operation.
+* ZBUFF_DCtx objects can be reused multiple times.
+*
+* Use ZBUFF_decompressContinue() repetitively to consume your input.
+* *srcSizePtr and *maxDstSizePtr can be any size.
+* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
+* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
+* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
+* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+* or 0 when a frame is completely decoded
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize
+* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
+* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
+* **************************************************/
+
+
+/* *************************************
+* Tool functions
+***************************************/
+unsigned ZBUFF_isError(size_t errorCode);
+const char* ZBUFF_getErrorName(size_t errorCode);
+
+/** The below functions provide recommended buffer sizes for Compression or Decompression operations.
+* These sizes are not compulsory, they just tend to offer better latency */
+size_t ZBUFF_recommendedCInSize(void);
+size_t ZBUFF_recommendedCOutSize(void);
+size_t ZBUFF_recommendedDInSize(void);
+size_t ZBUFF_recommendedDOutSize(void);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_BUFFERED_H */
--- /dev/null
+/*
+ zstd - buffered version of compression library
+ experimental complementary API, for static linking only
+ Copyright (C) 2015, Yann Collet.
+
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - zstd source repository : https://github.com/Cyan4973/zstd
+ - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#ifndef ZSTD_BUFFERED_STATIC_H
+#define ZSTD_BUFFERED_STATIC_H
+
+/* The objects defined into this file should be considered experimental.
+ * They are not labelled stable, as their prototype may change in the future.
+ * You can use them for tests, provide feedback, or if you can endure risk of future changes.
+ */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* *************************************
+* Includes
+***************************************/
+#include "zstd_static.h"
+#include "zstd_buffered.h"
+
+
+/* *************************************
+* Advanced Streaming functions
+***************************************/
+size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_BUFFERED_STATIC_H */
/** ZSTD_validateParams
correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */
-void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint)
+void ZSTD_validateParams(ZSTD_parameters* params)
{
const U32 btPlus = (params->strategy == ZSTD_btlazy2);
if (params->windowLog < ZSTD_WINDOWLOG_MIN) params->windowLog = ZSTD_WINDOWLOG_MIN;
/* correct params, to use less memory */
- if ((srcSizeHint > 0) && (srcSizeHint < (1<<ZSTD_WINDOWLOG_MAX)))
+ if ((params->srcSize > 0) && (params->srcSize < (1<<ZSTD_WINDOWLOG_MAX)))
{
- U32 srcLog = ZSTD_highbit((U32)srcSizeHint-1) + 1;
+ U32 srcLog = ZSTD_highbit((U32)(params->srcSize)-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog;
}
+ if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus; /* <= ZSTD_CONTENTLOG_MAX */
if (params->contentLog < ZSTD_CONTENTLOG_MIN) params->contentLog = ZSTD_CONTENTLOG_MIN;
if (params->hashLog > ZSTD_HASHLOG_MAX) params->hashLog = ZSTD_HASHLOG_MAX;
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
- ZSTD_parameters params,
- U64 srcSizeHint)
+ ZSTD_parameters params)
{
- ZSTD_validateParams(¶ms, srcSizeHint);
-
/* reserve table memory */
{
const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
}
+/** ZSTD_reduceIndex
+* rescale indexes to avoid future overflow (indexes are U32) */
static void ZSTD_reduceIndex (ZSTD_CCtx* zc,
const U32 reducerValue)
{
- const U32 contentLog = zc->params.strategy == ZSTD_fast ? 1 : zc->params.contentLog;
+ const U32 contentLog = (zc->params.strategy == ZSTD_fast) ? 1 : zc->params.contentLog;
const U32 tableSpaceU32 = (1 << contentLog) + (1 << zc->params.hashLog);
U32* table32 = zc->hashTable;
U32 index;
}
+/** ZSTD_compressBegin_advanced
+* Write frame header, according to params
+* @return : nb of bytes written */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize,
- const ZSTD_parameters params,
- const U64 srcSizeHint)
+ ZSTD_parameters params)
{
size_t errorCode;
- if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
- errorCode = ZSTD_resetCCtx_advanced(ctx, params, srcSizeHint);
+
+ ZSTD_validateParams(¶ms);
+
+ if (maxDstSize < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
+ errorCode = ZSTD_resetCCtx_advanced(ctx, params);
if (ZSTD_isError(errorCode)) return errorCode;
- MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */
- return 4;
+ MEM_writeLE32(dst, ZSTD_MAGICNUMBER); /* Write Header */
+ ((BYTE*)dst)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN);
+ return ZSTD_frameHeaderSize_min;
}
-size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint)
+/** ZSTD_getParams
+* return ZSTD_parameters structure for a selected compression level and srcSize.
+* srcSizeHint value is optional, select 0 if not known */
+ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint)
{
- int tableID = ((srcSizeHint-1) > 128 KB); /* intentional underflow for srcSizeHint == 0 */
+ ZSTD_parameters result;
+ int tableID = ((srcSizeHint-1) <= 128 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
- return ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_defaultParameters[tableID][compressionLevel], srcSizeHint);
+ result = ZSTD_defaultParameters[tableID][compressionLevel];
+ result.srcSize = srcSizeHint;
+ return result;
+}
+
+
+size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint)
+{
+ return ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_getParams(compressionLevel, srcSizeHint));
}
+/** ZSTD_compressEnd
+* Write frame epilogue
+* @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
{
BYTE* op = (BYTE*)dst;
}
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
- void* dst, size_t maxDstSize,
- const void* src, size_t srcSize,
- ZSTD_parameters params)
+ void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize,
+ ZSTD_parameters params)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
size_t oSize;
/* Header */
- oSize = ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize);
+ oSize = ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, params);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
maxDstSize -= oSize;
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
{
- const int tableID = (srcSize > 128 KB);
- if (compressionLevel < 1) compressionLevel = 1;
- if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
- return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_defaultParameters[tableID][compressionLevel]);
+ return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_getParams(compressionLevel, srcSize));
}
size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
/* *************************************************************
* Context management
***************************************************************/
+typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
+ ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage;
+
struct ZSTD_DCtx_s
{
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
void* vBase;
void* dictEnd;
size_t expected;
+ size_t headerSize;
+ ZSTD_parameters params;
blockType_t bType;
- U32 phase;
+ ZSTD_dStage stage;
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
+ BYTE headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
{
- dctx->expected = ZSTD_frameHeaderSize;
- dctx->phase = 0;
+ dctx->expected = ZSTD_frameHeaderSize_min;
+ dctx->stage = ZSTDds_getFrameHeaderSize;
dctx->previousDstEnd = NULL;
dctx->base = NULL;
dctx->vBase = NULL;
/* *************************************************************
* Decompression section
***************************************************************/
+/** ZSTD_decodeFrameHeader_Part1
+* decode the 1st part of the Frame Header, which tells Frame Header size.
+* srcSize must be == ZSTD_frameHeaderSize_min
+* @return : the full size of the Frame Header */
+static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_t srcSize)
+{
+ U32 magicNumber;
+ if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
+ magicNumber = MEM_readLE32(src);
+ if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
+ zc->headerSize = ZSTD_frameHeaderSize_min;
+ return zc->headerSize;
+}
+
+/** ZSTD_decodeFrameHeader_Part2
+* decode the full Frame Header
+* srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1
+* @return : 0, or an error code, which can be tested using ZSTD_isError() */
+static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_t srcSize)
+{
+ const BYTE* ip = (const BYTE*)src;
+ if (srcSize != zc->headerSize) return ERROR(srcSize_wrong);
+ memset(&(zc->params), 0, sizeof(zc->params));
+ zc->params.windowLog = ip[4] + ZSTD_WINDOWLOG_ABSOLUTEMIN;
+ return 0;
+}
+
+
+size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
+{
+ U32 magicNumber;
+ if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max;
+ magicNumber = MEM_readLE32(src);
+ if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
+ memset(params, 0, sizeof(*params));
+ params->windowLog = ((const BYTE*)src)[4] + ZSTD_WINDOWLOG_ABSOLUTEMIN;
+ return 0;
+}
+
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
{
BYTE* op = ostart;
BYTE* const oend = ostart + maxDstSize;
size_t remainingSize = srcSize;
- U32 magicNumber;
blockProperties_t blockProperties;
ctx->base = ctx->vBase = ctx->dictEnd = dst;
/* Frame Header */
- if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
- magicNumber = MEM_readLE32(src);
+ {
+ size_t frameHeaderSize;
+ if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
- if (ZSTD_isLegacy(magicNumber))
- return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber);
+ {
+ const U32 magicNumber = MEM_readLE32(src);
+ if (ZSTD_isLegacy(magicNumber))
+ return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber);
+ }
#endif
- if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
- ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
+ frameHeaderSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
+ if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
+ if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+ ip += frameHeaderSize; remainingSize -= frameHeaderSize;
+ frameHeaderSize = ZSTD_decodeFrameHeader_Part2(ctx, src, frameHeaderSize);
+ if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
+ }
/* Loop on each block */
while (1)
/* ******************************
* Streaming Decompression API
********************************/
-
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
{
return dctx->expected;
{
/* Sanity check */
if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
- if (dst != ctx->previousDstEnd) /* not contiguous */
+ if (dst != ctx->previousDstEnd) /* not contiguous */
{
ctx->dictEnd = ctx->previousDstEnd;
if ((dst > ctx->base) && (dst < ctx->previousDstEnd)) /* rolling buffer : new segment right into tracked memory */
ctx->base = dst;
}
- /* Decompress : frame header */
- if (ctx->phase == 0)
- {
- /* Check frame magic header */
- U32 magicNumber = MEM_readLE32(src);
- if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
- ctx->phase = 1;
- ctx->expected = ZSTD_blockHeaderSize;
- return 0;
- }
-
- /* Decompress : block header */
- if (ctx->phase == 1)
+ /* Decompress : frame header; part 1 */
+ switch (ctx->stage)
{
- blockProperties_t bp;
- size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
- if (ZSTD_isError(blockSize)) return blockSize;
- if (bp.blockType == bt_end)
+ case ZSTDds_getFrameHeaderSize :
{
- ctx->expected = 0;
- ctx->phase = 0;
+ /* get frame header size */
+ if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
+ ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
+ if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize;
+ memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
+ if (ctx->headerSize > ZSTD_frameHeaderSize_min)
+ {
+ ctx->expected = ctx->headerSize - ZSTD_frameHeaderSize_min;
+ ctx->stage = ZSTDds_decodeFrameHeader;
+ return 0;
+ }
+ ctx->expected = 0; /* not necessary to copy more */
}
- else
+ case ZSTDds_decodeFrameHeader:
{
- ctx->expected = blockSize;
- ctx->bType = bp.blockType;
- ctx->phase = 2;
+ /* get frame header */
+ size_t result;
+ memcpy(ctx->headerBuffer + ZSTD_frameHeaderSize_min, src, ctx->expected);
+ result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize);
+ if (ZSTD_isError(result)) return result;
+ ctx->expected = ZSTD_blockHeaderSize;
+ ctx->stage = ZSTDds_decodeBlockHeader;
+ return 0;
}
+ case ZSTDds_decodeBlockHeader:
+ {
+ /* Decode block header */
+ blockProperties_t bp;
+ size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
+ if (ZSTD_isError(blockSize)) return blockSize;
+ if (bp.blockType == bt_end)
+ {
+ ctx->expected = 0;
+ ctx->stage = ZSTDds_getFrameHeaderSize;
+ }
+ else
+ {
+ ctx->expected = blockSize;
+ ctx->bType = bp.blockType;
+ ctx->stage = ZSTDds_decompressBlock;
+ }
- ctx->previousDstEnd = dst;
- return 0;
- }
-
- /* Decompress : block content */
- {
- size_t rSize;
- switch(ctx->bType)
+ ctx->previousDstEnd = dst;
+ return 0;
+ }
+ case 3:
{
- case bt_compressed:
- rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
- break;
- case bt_raw :
- rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
- break;
- case bt_rle :
- return ERROR(GENERIC); /* not yet handled */
- break;
- case bt_end : /* should never happen (filtered at phase 1) */
- rSize = 0;
- break;
- default:
- return ERROR(GENERIC);
+ /* Decompress : block content */
+ size_t rSize;
+ switch(ctx->bType)
+ {
+ case bt_compressed:
+ rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
+ break;
+ case bt_raw :
+ rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
+ break;
+ case bt_rle :
+ return ERROR(GENERIC); /* not yet handled */
+ break;
+ case bt_end : /* should never happen (filtered at phase 1) */
+ rSize = 0;
+ break;
+ default:
+ return ERROR(GENERIC);
+ }
+ ctx->stage = ZSTDds_decodeBlockHeader;
+ ctx->expected = ZSTD_blockHeaderSize;
+ ctx->previousDstEnd = (char*)dst + rSize;
+ return rSize;
}
- ctx->phase = 1;
- ctx->expected = ZSTD_blockHeaderSize;
- ctx->previousDstEnd = (char*)dst + rSize;
- return rSize;
+ default:
+ return ERROR(GENERIC); /* impossible */
}
-
}
/* *************************************
* Common constants
***************************************/
+#define ZSTD_MAGICNUMBER 0xFD2FB524 /* v0.4 */
+
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define BLOCKSIZE (128 KB) /* define, for static allocation */
static const size_t ZSTD_blockHeaderSize = 3;
-static const size_t ZSTD_frameHeaderSize = 4;
+static const size_t ZSTD_frameHeaderSize_min = 5;
+#define ZSTD_frameHeaderSize_max 5 /* define, for static allocation */
#define BIT7 128
#define BIT6 64
/* *************************************
* Types
***************************************/
+#define ZSTD_WINDOWLOG_MAX 26
+#define ZSTD_WINDOWLOG_MIN 18
+#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
+#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
+#define ZSTD_CONTENTLOG_MIN 4
+#define ZSTD_HASHLOG_MAX 28
+#define ZSTD_HASHLOG_MIN 4
+#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
+#define ZSTD_SEARCHLOG_MIN 1
+#define ZSTD_SEARCHLENGTH_MAX 7
+#define ZSTD_SEARCHLENGTH_MIN 4
+
/** from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
typedef struct
{
- U32 windowLog; /* largest match distance : impact decompression buffer size */
+ U64 srcSize; /* optional : tells how much bytes are present in the frame. Use 0 if not known. */
+ U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
- U32 hashLog; /* dispatch table : larger == more memory, faster*/
- U32 searchLog; /* nb of searches : larger == more compression, slower*/
- U32 searchLength; /* size of matches : larger == faster decompression */
+ U32 hashLog; /* dispatch table : larger == more memory, faster */
+ U32 searchLog; /* nb of searches : larger == more compression, slower */
+ U32 searchLength; /* size of matches : larger == faster decompression, sometimes less compression */
ZSTD_strategy strategy;
} ZSTD_parameters;
/* *************************************
* Advanced function
***************************************/
+/** ZSTD_getParams
+* return ZSTD_parameters structure for a selected compression level and srcSize.
+* srcSizeHint value is optional, select 0 if not known */
+ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint);
+
+/** ZSTD_validateParams
+* correct params value to remain within authorized range */
+void ZSTD_validateParams(ZSTD_parameters* params);
+
/** ZSTD_compress_advanced
* Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
- void* dst, size_t maxDstSize,
- const void* src, size_t srcSize,
- ZSTD_parameters params);
-
-/** ZSTD_validateParams
- correct params value to remain within authorized range
- srcSizeHint value is optional, select 0 if not known */
-void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint);
+ void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize,
+ ZSTD_parameters params);
-/* *************************************
-* Streaming functions
-***************************************/
+/* **************************************
+* Streaming functions (bufferless mode)
+****************************************/
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
+
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
+size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
-/*
- Use above functions alternatively.
+/**
+ Streaming decompression, bufferless mode
+
+ A ZSTD_DCtx object is required to track streaming operations.
+ Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
+ A ZSTD_DCtx object can be re-used multiple times. Use ZSTD_resetDCtx() to return to fresh status.
+
+ First operation is to retrieve frame parameters, using ZSTD_getFrameParams().
+ This function doesn't consume its input. It needs enough input data to properly decode the frame header.
+ The objective is to retrieve *params.windowlog, to know how much memory is required during decoding.
+ Result : 0 if successfull, it means the ZSTD_parameters structure has been filled.
+ >0 : means there is not enough data into src. Provides the expected size to successfully decode header.
+ errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header)
+
+ Then it's possible to start decompression.
+ Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
- ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.
- Result is the number of bytes regenerated within 'dst'.
+ ZSTD_decompressContinue() will use previous data blocks during decompress.
+ They should be located contiguously prior to current block. Alternatively, a round buffer is possible.
+ Just make sure that the combined of current and accessible past blocks is a minimum of (1 << windowlog).
+
+ @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst'.
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
-*/
-/* *************************************
-* Prefix - version detection
-***************************************/
-#define ZSTD_magicNumber 0xFD2FB523 /* v0.3 (current)*/
+ A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
+*/
/* *************************************
* Pre-defined compression levels
***************************************/
#define ZSTD_MAX_CLEVEL 20
-#define ZSTD_WINDOWLOG_MAX 26
-#define ZSTD_WINDOWLOG_MIN 18
-#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
-#define ZSTD_CONTENTLOG_MIN 4
-#define ZSTD_HASHLOG_MAX 28
-#define ZSTD_HASHLOG_MIN 4
-#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
-#define ZSTD_SEARCHLOG_MIN 1
-#define ZSTD_SEARCHLENGTH_MAX 7
-#define ZSTD_SEARCHLENGTH_MIN 4
-
static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = {
-{ /* for <= 128 KB */
- /* W, C, H, S, L, strat */
- { 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
- { 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */
- { 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */
- { 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */
- { 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */
- { 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */
- { 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */
- { 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */
- { 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */
- { 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */
- { 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */
- { 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */
- { 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */
- { 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */
- { 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */
- { 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */
- { 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */
- { 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */
- { 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */
- { 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */
- { 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */
+{ /* "default" */
+ /* W, C, H, S, L, strat */
+ { 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
+ { 0, 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */
+ { 0, 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */
+ { 0, 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */
+ { 0, 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */
+ { 0, 20, 14, 18, 3, 5, ZSTD_greedy }, /* level 5 */
+ { 0, 20, 18, 19, 3, 5, ZSTD_greedy }, /* level 6 */
+ { 0, 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */
+ { 0, 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */
+ { 0, 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */
+ { 0, 21, 19, 21, 4, 5, ZSTD_lazy2 }, /* level 10 */
+ { 0, 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */
+ { 0, 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */
+ { 0, 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */
+ { 0, 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */
+ { 0, 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */
+ { 0, 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */
+ { 0, 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */
+ { 0, 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */
+ { 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
+ { 0, 25, 26, 25, 6, 5, ZSTD_btlazy2 }, /* level 20 */
},
-{ /* for > 128 KB */
+{ /* for srcSize <= 128 KB */
/* W, C, H, S, L, strat */
- { 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
- { 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */
- { 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */
- { 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */
- { 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */
- { 20, 14, 18, 3, 5, ZSTD_greedy }, /* level 5 */
- { 20, 18, 19, 3, 5, ZSTD_greedy }, /* level 6 */
- { 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */
- { 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */
- { 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */
- { 21, 19, 21, 4, 5, ZSTD_lazy2 }, /* level 10 */
- { 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */
- { 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */
- { 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */
- { 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */
- { 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */
- { 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */
- { 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */
- { 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */
- { 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
- { 25, 26, 25, 6, 5, ZSTD_btlazy2 }, /* level 20 */
-}
+ { 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
+ { 0, 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */
+ { 0, 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */
+ { 0, 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */
+ { 0, 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */
+ { 0, 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */
+ { 0, 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */
+ { 0, 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */
+ { 0, 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */
+ { 0, 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */
+ { 0, 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */
+ { 0, 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */
+ { 0, 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */
+ { 0, 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */
+ { 0, 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */
+ { 0, 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */
+ { 0, 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */
+ { 0, 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */
+ { 0, 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */
+ { 0, 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */
+ { 0, 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */
+},
};
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
-zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
-zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-
#define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */
#define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */
#include <sys/stat.h> /* stat64 */
#include "mem.h"
#include "fileio.h"
-#include "zstd_static.h"
+#include "zstd_static.h" /* ZSTD_magicNumber */
+#include "zstd_buffered_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-# include "zstd_legacy.h" /* legacy */
+# include "zstd_legacy.h" /* legacy */
# include "fileio_legacy.h" /* legacy */
#endif
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# ifdef __MINGW32__
- /* int _fileno(FILE *stream); // seems no longer useful // MINGW somehow forgets to include this windows declaration into <stdio.h> */
+ // int _fileno(FILE *stream); /* seems no longer useful /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
# endif
# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#define BIT6 0x40
#define BIT7 0x80
-//static const unsigned FIO_maxBlockSizeID = 0xB; /* => 2MB block */
-static const unsigned FIO_blockHeaderSize = 3;
+#define BLOCKSIZE (128 KB)
+#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define FSE_CHECKSUM_SEED 0
U64 filesize = 0;
U64 compressedfilesize = 0;
BYTE* inBuff;
- BYTE* inSlot;
- BYTE* inEnd;
BYTE* outBuff;
- size_t blockSize = 128 KB;
- size_t inBuffSize = 4 * blockSize;
- size_t outBuffSize = ZSTD_compressBound(blockSize);
+ size_t inBuffSize = ZBUFF_recommendedCInSize();
+ size_t outBuffSize = ZBUFF_recommendedCOutSize();
FILE* finput;
FILE* foutput;
- size_t sizeCheck, cSize;
- ZSTD_CCtx* ctx;
-
- /* init */
- FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
- filesize = FIO_getFileSize(input_filename);
+ size_t sizeCheck, errorCode;
+ ZBUFF_CCtx* ctx;
/* Allocate Memory */
- ctx = ZSTD_createCCtx();
+ ctx = ZBUFF_createCCtx();
inBuff = (BYTE*)malloc(inBuffSize);
outBuff = (BYTE*)malloc(outBuffSize);
if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
- inSlot = inBuff;
- inEnd = inBuff + inBuffSize;
-
- /* Write Frame Header */
- cSize = ZSTD_compressBegin(ctx, outBuff, outBuffSize, cLevel, filesize);
- if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
- sizeCheck = fwrite(outBuff, 1, cSize, foutput);
- if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
- compressedfilesize += cSize;
+ /* init */
+ FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
+ filesize = FIO_getFileSize(input_filename);
+ errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize));
+ if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression");
filesize = 0;
/* Main compression loop */
size_t inSize;
/* Fill input Buffer */
- if (inSlot + blockSize > inEnd) inSlot = inBuff;
- inSize = fread(inSlot, (size_t)1, blockSize, finput);
+ inSize = fread(inBuff, (size_t)1, inBuffSize, finput);
if (inSize==0) break;
filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
- /* Compress Block */
- cSize = ZSTD_compressContinue(ctx, outBuff, outBuffSize, inSlot, inSize);
- if (ZSTD_isError(cSize))
- EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize));
-
- /* Write cBlock */
- sizeCheck = fwrite(outBuff, 1, cSize, foutput);
- if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
- compressedfilesize += cSize;
- inSlot += inSize;
+ {
+ /* Compress (buffered streaming ensures appropriate formatting) */
+ size_t usedInSize = inSize;
+ size_t cSize = outBuffSize;
+ size_t result = ZBUFF_compressContinue(ctx, outBuff, &cSize, inBuff, &usedInSize);
+ if (ZBUFF_isError(result))
+ EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result));
+ if (inSize != usedInSize)
+ /* inBuff should be entirely consumed since buffer sizes are recommended ones */
+ EXM_THROW(24, "Compression error : input block not fully consumed");
+
+ /* Write cBlock */
+ sizeCheck = fwrite(outBuff, 1, cSize, foutput);
+ if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
+ compressedfilesize += cSize;
+ }
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
}
/* End of Frame */
- cSize = ZSTD_compressEnd(ctx, outBuff, outBuffSize);
- if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
+ {
+ size_t cSize = outBuffSize;
+ size_t result = ZBUFF_compressEnd(ctx, outBuff, &cSize);
+ if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
- sizeCheck = fwrite(outBuff, 1, cSize, foutput);
- if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
- compressedfilesize += cSize;
+ sizeCheck = fwrite(outBuff, 1, cSize, foutput);
+ if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
+ compressedfilesize += cSize;
+ }
/* Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
/* clean */
free(inBuff);
free(outBuff);
- ZSTD_freeCCtx(ctx);
+ ZBUFF_freeCCtx(ctx);
fclose(finput);
if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
- BYTE* inBuff, size_t inBuffSize,
+ BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded,
BYTE* outBuff, size_t outBuffSize,
- ZSTD_DCtx* dctx)
+ ZBUFF_DCtx* dctx)
{
- BYTE* op = outBuff;
- BYTE* const oend = outBuff + outBuffSize;
- U64 filesize = 0;
- size_t toRead;
- size_t sizeCheck;
-
+ U64 frameSize = 0;
+ size_t readSize=alreadyLoaded;
/* Main decompression Loop */
- toRead = ZSTD_nextSrcSizeToDecompress(dctx);
- while (toRead)
+ ZBUFF_decompressInit(dctx);
+ while (1)
{
- size_t readSize, decodedSize;
+ /* Decode */
+ size_t sizeCheck;
+ size_t inSize=readSize, decodedSize=outBuffSize;
+ size_t inStart=0;
+ size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize);
+ if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
+ if (toRead==0) break; /* end of Frame */
+ readSize -= inSize;
+ inStart += inSize;
+
+ /* Write block */
+ sizeCheck = fwrite(outBuff, 1, decodedSize, foutput);
+ if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
+ frameSize += decodedSize;
+ DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
+
+ if (readSize) continue; /* still some data left within inBuff */
/* Fill input buffer */
- if (toRead > inBuffSize)
- EXM_THROW(34, "too large block");
+ if (toRead > inBuffSize) EXM_THROW(34, "too large block");
readSize = fread(inBuff, 1, toRead, finput);
- if (readSize != toRead)
- EXM_THROW(35, "Read error");
-
- /* Decode block */
- decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
- if (ZSTD_isError(decodedSize)) EXM_THROW(36, "Decoding error : input corrupted");
-
- if (decodedSize) /* not a header */
- {
- /* Write block */
- sizeCheck = fwrite(op, 1, decodedSize, foutput);
- if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
- filesize += decodedSize;
- op += decodedSize;
- if (op==oend) op = outBuff;
- DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) );
- }
-
- /* prepare for next Block */
- toRead = ZSTD_nextSrcSizeToDecompress(dctx);
+ if (readSize != toRead) EXM_THROW(35, "Read error");
}
- return filesize;
+ return frameSize;
}
-#define MAXHEADERSIZE (FIO_FRAMEHEADERSIZE+3)
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
{
FILE* finput, *foutput;
BYTE* inBuff=NULL;
- size_t inBuffSize = 0;
+ size_t inBuffSize = ZBUFF_recommendedDInSize();
BYTE* outBuff=NULL;
- size_t outBuffSize = 0;
- U32 blockSize = 128 KB;
- U32 wNbBlocks = 4;
+ size_t outBuffSize = ZBUFF_recommendedDOutSize();
U64 filesize = 0;
- BYTE* header[MAXHEADERSIZE];
size_t toRead;
size_t sizeCheck;
/* Init */
- ZSTD_DCtx* dctx = ZSTD_createDCtx();
+ ZBUFF_DCtx* dctx = ZBUFF_createDCtx();
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
+ /* Allocate Memory (if needed) */
+ inBuff = (BYTE*)malloc(inBuffSize);
+ outBuff = (BYTE*)malloc(outBuffSize);
+ if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
+
/* for each frame */
for ( ; ; )
{
- /* check magic number -> version */
U32 magicNumber;
- toRead = sizeof(ZSTD_magicNumber);;
- sizeCheck = fread(header, (size_t)1, toRead, finput);
- if (sizeCheck==0) break; /* no more input */
- if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
+ toRead = 0;
- magicNumber = MEM_readLE32(header);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+ /* check magic number -> version */
+ toRead = 4;
+ sizeCheck = fread(inBuff, (size_t)1, toRead, finput);
+ if (sizeCheck==0) break; /* no more input */
+ if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
+ magicNumber = MEM_readLE32(inBuff);
if (ZSTD_isLegacy(magicNumber))
{
filesize += FIO_decompressLegacyFrame(foutput, finput, magicNumber);
continue;
}
#endif /* ZSTD_LEGACY_SUPPORT */
- if (magicNumber != ZSTD_magicNumber) EXM_THROW(32, "Error : unknown frame prefix");
-
- /* prepare frame decompression, by completing header */
- ZSTD_resetDCtx(dctx);
- toRead = ZSTD_nextSrcSizeToDecompress(dctx) - sizeof(ZSTD_magicNumber);
- if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
- sizeCheck = fread(&header[sizeof(ZSTD_magicNumber)], 1, toRead, finput);
- if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
- sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, sizeof(ZSTD_magicNumber)+toRead); // Decode frame header
- if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
-
- /* Here later : blockSize determination */
-
- /* Allocate Memory (if needed) */
- {
- size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
- size_t newOutBuffSize = wNbBlocks * blockSize;
- if (newInBuffSize > inBuffSize)
- {
- free(inBuff);
- inBuffSize = newInBuffSize;
- inBuff = (BYTE*)malloc(inBuffSize);
- }
- if (newOutBuffSize > outBuffSize)
- {
- free(outBuff);
- outBuffSize = newOutBuffSize;
- outBuff = (BYTE*)malloc(outBuffSize);
- }
- }
- if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
- filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
+ filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, toRead, outBuff, outBuffSize, dctx);
}
DISPLAYLEVEL(2, "\r%79s\r", "");
/* clean */
free(inBuff);
free(outBuff);
- ZSTD_freeDCtx(dctx);
+ ZBUFF_freeDCtx(dctx);
fclose(finput);
if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
}
-#if 0
-unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
-{
- FILE* finput, *foutput;
- BYTE* inBuff=NULL;
- size_t inBuffSize = 0;
- BYTE* outBuff=NULL;
- size_t outBuffSize = 0;
- U32 blockSize = 128 KB;
- U32 wNbBlocks = 4;
- U64 filesize = 0;
- BYTE* header[MAXHEADERSIZE];
- ZSTD_Dctx* dctx;
- size_t toRead;
- size_t sizeCheck;
-
-
- /* Init */
- FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
- dctx = ZSTD_createDCtx();
-
- /* for each frame */
- for ( ; ; )
- {
- /* check header */
- ZSTD_resetDCtx(dctx);
- toRead = ZSTD_nextSrcSizeToDecompress(dctx);
- if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
- sizeCheck = fread(header, (size_t)1, toRead, finput);
- if (sizeCheck==0) break; /* no more input */
- if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
- sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
- if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
-
- /* Here later : blockSize determination */
-
- /* Allocate Memory (if needed) */
- {
- size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
- size_t newOutBuffSize = wNbBlocks * blockSize;
- if (newInBuffSize > inBuffSize)
- {
- free(inBuff);
- inBuffSize = newInBuffSize;
- inBuff = (BYTE*)malloc(inBuffSize);
- }
- if (newOutBuffSize > outBuffSize)
- {
- free(outBuff);
- outBuffSize = newOutBuffSize;
- outBuff = (BYTE*)malloc(outBuffSize);
- }
- }
- if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
-
- filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
- }
-
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "Decoded %llu bytes \n", (long long unsigned)filesize);
-
- /* clean */
- free(inBuff);
- free(outBuff);
- ZSTD_freeDCtx(dctx);
- fclose(finput);
- if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
-
- return filesize;
-}
-#endif
return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
}
-size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
-{
- U32 i;
- size_t total = 0;
- BYTE* data = (BYTE*)buff2;
-
- (void)dst; (void)dstSize; (void)src;
- for (i=0; i < srcSize; i++)
- {
- U32 b = data[i];
- total += b;
- if (b==0) total = 0; // 825
- //if (!b) total = 0; // 825
- //total = b ? total : 0; // 622
- //total &= -!b; // 622
- //total *= !!b; // 465
- }
- return total;
-}
-
-size_t local_decodeLiteralsForward(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
-{
- (void)src; (void)srcSize;
- return FSE_decompress(dst, dstSize, buff2, g_cSize);
-}
case 32:
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
break;
- case 101:
- benchFunction = local_conditionalNull; benchName = "conditionalNull";
- break;
- case 102:
- benchFunction = local_decodeLiteralsForward; benchName = "ZSTD_decodeLiteralsForward";
- break;
default :
return 0;
}
{
blockProperties_t bp;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
- ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block type
+ ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
goto _cleanOut;
}
- memcpy(buff2, dstBuff+7, g_cSize-7);
- srcSize = srcSize > 128 KB ? 128 KB : srcSize; // relative to block
+ memcpy(buff2, dstBuff+8, g_cSize-8);
+ srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break;
}
case 32: /* ZSTD_decodeSeqHeaders */
const BYTE* ip = dstBuff;
const BYTE* iend;
size_t blockSize;
- ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
- ip += 4; // Jump magic Number
- blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block type
+ ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
+ ip += 5; /* Skip frame Header */
+ blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
}
iend = ip + 3 + blockSize; /* End of first block */
ip += 3; /* skip block header */
- ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip); // jump literal sub block and its header
+ ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip); /* skip literal segment */
g_cSize = iend-ip;
- memcpy(buff2, ip, g_cSize); // copy rest of block (starting with SeqHeader)
- srcSize = srcSize > 128 KB ? 128 KB : srcSize; // speed relative to block
+ memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
+ srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break;
}
/* test functions */
+ /* by convention, test functions can be added > 100 */
- case 101: /* conditionalNull */
- {
- size_t i;
- for (i=0; i<srcSize; i++)
- buff2[i] = i & 15;
- break;
- }
- case 102: /* local_decodeLiteralsForward */
- {
- blockProperties_t bp;
- ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
- g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp);
- memcpy(buff2, dstBuff+10, g_cSize);
- //srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
- srcSize = srcSize > 128 KB ? 128 KB : srcSize; // relative to block
- break;
- }
default : ;
}
#if defined(_MSC_VER)
# define snprintf _snprintf /* snprintf unsupported by Visual <= 2012 */
#endif
-
+
/**************************************
* Includes
static U32 g_singleRun = 0;
static U32 g_target = 0;
static U32 g_noSeed = 0;
-static const ZSTD_parameters* g_seedParams = ZSTD_defaultParameters[0];
-static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_greedy };
+static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
void BMK_SetNbIterations(int nbLoops)
{
* Private functions
*********************************************************/
-static unsigned BMK_highbit(U32 val)
-{
-# if defined(_MSC_VER) /* Visual */
- unsigned long r;
- _BitScanReverse(&r, val);
- return (unsigned)r;
-# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
- return 31 - __builtin_clz(val);
-# else /* Software version */
- static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
- U32 v = val;
- int r;
- v |= v >> 1;
- v |= v >> 2;
- v |= v >> 4;
- v |= v >> 8;
- v |= v >> 16;
- r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
- return r;
-# endif
-}
-
#if defined(BMK_LEGACY_TIMER)
static int BMK_GetMilliStart(void)
/* validate new conf */
{
ZSTD_parameters saved = p;
- ZSTD_validateParams(&p, g_blockSize ? g_blockSize : srcSize);
+ ZSTD_validateParams(&p);
if (memcmp(&p, &saved, sizeof(p))) continue; /* p was invalid */
}
const char* rfName = "grillResults.txt";
FILE* f;
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
- const U32 srcLog = BMK_highbit((U32)(blockSize-1))+1;
if (g_singleRun)
{
BMK_result_t testResult;
- ZSTD_validateParams(&g_params, blockSize);
+ g_params.srcSize = blockSize;
+ ZSTD_validateParams(&g_params);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
DISPLAY("\n");
return;
params.searchLog = 1;
params.searchLength = 7;
params.strategy = ZSTD_fast;
- ZSTD_validateParams(¶ms, blockSize);
+ params.srcSize = blockSize;
+ ZSTD_validateParams(¶ms);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
- g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4;
+ g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5;
}
/* establish speed objectives (relative to level 1) */
/* populate initial solution */
{
- const int tableID = (blockSize > 128 KB);
const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL;
- g_seedParams = ZSTD_defaultParameters[tableID];
for (i=1; i<=maxSeeds; i++)
{
- const U32 btPlus = (params.strategy == ZSTD_btlazy2);
- params = g_seedParams[i];
- params.windowLog = MIN(srcLog, params.windowLog);
- params.contentLog = MIN(params.windowLog+btPlus, params.contentLog);
- params.searchLog = MIN(params.contentLog, params.searchLog);
+ params = ZSTD_getParams(i, blockSize);
BMK_seed(winners, params, srcBuffer, srcSize, ctx);
}
}
case 'S':
g_singleRun = 1;
argument++;
- g_params = g_seedParams[2];
+ g_params = ZSTD_getParams(2, g_blockSize);
for ( ; ; )
{
switch(*argument)
argument++;
while ((*argument>= '0') && (*argument<='9'))
cLevel *= 10, cLevel += *argument++ - '0';
- g_params = g_seedParams[cLevel];
+ g_params = ZSTD_getParams(cLevel, g_blockSize);
continue;
}
default : ;
projects / thirdparty / zstd.git / commitdiff
