From: Yann Collet Date: Tue, 13 Dec 2016 15:39:36 +0000 (+0100) Subject: Added ZSTD_initCStream_srcSize(). X-Git-Tag: v1.1.2~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e795c8a5f6cf07e913bf0766e44e4b8a5fad9ef0;p=thirdparty%2Fzstd.git Added ZSTD_initCStream_srcSize(). Added relevant test cases in zstreamtest --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 6e52ec8ec..3d10fbd96 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2852,6 +2852,8 @@ struct ZSTD_CStream_s { ZSTD_cStreamStage stage; U32 checksum; U32 frameEnded; + U64 pledgedSrcSize; + U64 inputProcessed; ZSTD_parameters params; ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CStream within "zstd.h" */ @@ -2909,6 +2911,8 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; zcs->stage = zcss_load; zcs->frameEnded = 0; + zcs->pledgedSrcSize = pledgedSrcSize; + zcs->inputProcessed = 0; return 0; /* ready to go */ } @@ -2961,6 +2965,12 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0); } +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); + return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); +} + size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) { return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel); @@ -3057,6 +3067,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, *srcSizePtr = ip - istart; *dstCapacityPtr = op - ostart; + zcs->inputProcessed += *srcSizePtr; if (zcs->frameEnded) return 0; { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; if (hintInSize==0) hintInSize = zcs->blockSize; @@ -3101,6 +3112,9 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) BYTE* const oend = (BYTE*)(output->dst) + output->size; BYTE* op = ostart; + if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize)) + return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */ + if (zcs->stage != zcss_final) { /* flush whatever remains */ size_t srcSize = 0; diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h index 8296dc64a..85f973557 100644 --- a/lib/deprecated/zbuff.h +++ b/lib/deprecated/zbuff.h @@ -27,7 +27,7 @@ extern "C" { * Dependencies ***************************************/ #include /* size_t */ -#include /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */ +#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */ /* *************************************************************** @@ -170,7 +170,6 @@ ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(voi #ifdef ZBUFF_STATIC_LINKING_ONLY - #ifndef ZBUFF_STATIC_H_30298098432 #define ZBUFF_STATIC_H_30298098432 @@ -203,7 +202,6 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv #endif /* ZBUFF_STATIC_H_30298098432 */ - #endif /* ZBUFF_STATIC_LINKING_ONLY */ diff --git a/lib/zstd.h b/lib/zstd.h index 44aa99566..a4766e335 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -237,20 +237,20 @@ typedef struct ZSTD_outBuffer_s { * * Start a new compression by initializing ZSTD_CStream. * Use ZSTD_initCStream() to start a new compression operation. -* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. +* Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section) * * Use ZSTD_compressStream() repetitively to consume input stream. * The function will automatically update both `pos` fields. * Note that it may not consume the entire input, in which case `pos < size`, * and it's up to the caller to present again remaining data. * @return : a size hint, preferred nb of bytes to use as input for next function call -* (it's just a hint, to help latency a little, any other value will work fine) -* (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() ) * or an error code, which can be tested using ZSTD_isError(). +* Note 1 : it's just a hint, to help latency a little, any other value will work fine. +* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() * -* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). +* At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream(). * `output->pos` will be updated. -* Note some content might still be left within internal buffer if `output->size` is too small. +* Note that some content might still be left within internal buffer if `output->size` is too small. * @return : nb of bytes still present within internal buffer (0 if it's empty) * or an error code, which can be tested using ZSTD_isError(). * @@ -259,15 +259,15 @@ typedef struct ZSTD_outBuffer_s { * The epilogue is required for decoders to consider a frame completed. * Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. * In which case, call again ZSTD_endStream() to complete the flush. -* @return : nb of bytes still present within internal buffer (0 if it's empty) +* @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed) * or an error code, which can be tested using ZSTD_isError(). * * *******************************************************************/ -/*===== Streaming compression functions ======*/ typedef struct ZSTD_CStream_s ZSTD_CStream; ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); + ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); @@ -300,10 +300,10 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame. * *******************************************************************************/ -/*===== Streaming decompression functions =====*/ typedef struct ZSTD_DStream_s ZSTD_DStream; ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); + ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); @@ -490,6 +490,7 @@ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); /*===== Advanced Streaming compression functions =====*/ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ @@ -602,7 +603,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). This information is not required to properly decode a frame. - == Special case : skippable frames == + == Special case : skippable frames == Skippable frames allow integration of user-defined data into a flow of concatenated frames. Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows : diff --git a/tests/Makefile b/tests/Makefile index 6110465f6..d7d25026a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -100,9 +100,9 @@ fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated -zbufftest : CFLAGS += -Wno-deprecated-declarations +zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c - $(CC) $(FLAGS) $^ -o $@$(EXT) # flag required to silence deprecation warnings + $(CC) $(FLAGS) $^ -o $@$(EXT) zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32 diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 9dc164eaf..14b739233 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -28,8 +28,8 @@ #include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ #include "zstd.h" /* ZSTD_compressBound */ -#define ZBUFF_STATIC_LINKING_ONLY -#include "zbuff.h" /* ZBUFF_createCCtx_advanced */ +#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */ +#include "zbuff.h" /* ZBUFF_isError */ #include "datagen.h" /* RDG_genBuffer */ #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /* XXH64_* */ @@ -265,11 +265,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres static const U32 maxSrcLog = 24; static const U32 maxSampleLog = 19; BYTE* cNoiseBuffer[5]; - size_t srcBufferSize = (size_t)1<> 5; } -/* -static unsigned FUZ_highbit32(U32 v32) -{ - unsigned nbBits = 0; - if (v32==0) return 0; - for ( ; v32 ; v32>>=1) nbBits++; - return nbBits; -} -*/ - static void* allocFunction(void* opaque, size_t size) { void* address = malloc(size); @@ -254,6 +245,38 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } } DISPLAYLEVEL(4, "OK \n"); + /* _srcSize compression test */ + DISPLAYLEVEL(4, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZSTD_initCStream_srcSize(zc, 1, CNBufferSize); + outBuff.dst = (char*)(compressedBuffer)+cSize; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); + if (ZSTD_isError(r)) goto _output_error; } + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + /* wrong _srcSize compression test */ + DISPLAYLEVEL(4, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); + ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1); + outBuff.dst = (char*)(compressedBuffer)+cSize; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); + if (ZSTD_isError(r)) goto _output_error; } + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */ + DISPLAYLEVEL(4, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); } + /* Complex context re-use scenario */ DISPLAYLEVEL(4, "test%3i : context re-use : ", testNb++); ZSTD_freeCStream(zc); @@ -519,7 +542,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; U32 n; for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) { - /* compress random chunk into random size dst buffer */ + /* compress random chunks into randomly sized dst buffers */ { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize); size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);