From: Yann Collet Date: Thu, 28 Jul 2016 13:29:08 +0000 (+0200) Subject: Changed ZSTD_compressEnd() X-Git-Tag: v0.8.0^2~24 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=62470b4bab17da9b2d498fe9d943e07f759bf5f8;p=thirdparty%2Fzstd.git Changed ZSTD_compressEnd() --- diff --git a/NEWS b/NEWS index 1a329d85c..7221e461c 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ v0.8.0 -New : updated compresson format Improved : better speed on clang and gcc -O2, thanks to Eric Biggers +Changed : modified API : ZSTD_compressEnd() Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers Fixed : statistics for large dictionaries (> 128 KB), reported by Ilona Papava diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 31c859092..29fd4dbb1 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -306,7 +306,7 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) } /* create epilogue */ zbc->stage = ZBUFFcs_final; - zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */ + zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize, NULL, 0); /* epilogue into outBuff */ } /* flush epilogue */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1605e8bf9..b69dffa25 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2287,7 +2287,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, op += cSize; } - if (lastFrameChunk) cctx->stage = ZSTDcs_ending; + if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; ZSTD_statsPrint(stats, cctx->params.cParams.searchLength); /* debug only */ return op-ostart; } @@ -2401,19 +2401,6 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, } -size_t ZSTD_compressContinueThenEnd (ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - size_t endResult; - size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); - if (ZSTD_isError(cSize)) return cSize; - endResult = ZSTD_compressEnd(cctx, (char*)dst + cSize, dstCapacity-cSize); - if (ZSTD_isError(endResult)) return endResult; - return cSize + endResult; -} - - size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx) { return MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); @@ -2593,10 +2580,10 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) } -/*! ZSTD_compressEnd() : -* Write frame epilogue. +/*! ZSTD_writeEpilogue() : +* Ends a frame. * @return : nb of bytes written into dst (or an error code) */ -size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) { BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; @@ -2634,6 +2621,19 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) } +size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); + if (ZSTD_isError(cSize)) return cSize; + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); + if (ZSTD_isError(endResult)) return endResult; + return cSize + endResult; +} + + /*! ZSTD_compress_usingPreparedCCtx() : * Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. * It avoids reloading the dictionary each time. @@ -2643,12 +2643,10 @@ static size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - { size_t const errorCode = ZSTD_copyCCtx(cctx, preparedCCtx); - if (ZSTD_isError(errorCode)) return errorCode; - } - { size_t const cSize = ZSTD_compressContinueThenEnd(cctx, dst, dstCapacity, src, srcSize); - return cSize; - } + size_t const errorCode = ZSTD_copyCCtx(cctx, preparedCCtx); + if (ZSTD_isError(errorCode)) return errorCode; + + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); } @@ -2658,16 +2656,10 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params) { - BYTE* const ostart = (BYTE*)dst; - BYTE* op = ostart; - - /* Init */ - { size_t const errorCode = ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize); - if(ZSTD_isError(errorCode)) return errorCode; } + size_t const errorCode = ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize); + if(ZSTD_isError(errorCode)) return errorCode; - /* body (compression) */ - { size_t const oSize = ZSTD_compressContinueThenEnd(cctx, op, dstCapacity, src, srcSize); - return oSize; } + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); } size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, diff --git a/lib/zstd.h b/lib/zstd.h index 9356f56d2..e3663e8b7 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -315,15 +315,14 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); /* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. * But it's also a complex one, with a lot of restrictions (documented below). * For an easier streaming API, look into common/zbuff.h -* which removes all restrictions by implementing and managing its own internal buffer */ +* which removes all restrictions by allocating and managing its own internal buffer */ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity); -ZSTDLIB_API size_t ZSTD_compressContinueThenEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /* A ZSTD_CCtx object is required to track streaming operations. @@ -338,7 +337,7 @@ ZSTDLIB_API size_t ZSTD_compressContinueThenEnd(ZSTD_CCtx* cctx, void* dst, size Then, consume your input using ZSTD_compressContinue(). There are some important considerations to keep in mind when using this advanced function : - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only. - - Interface is synchronous : input is consumed entirely and produce 1 (or more) compressed blocks. + - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks. - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. Worst case evaluation is provided by ZSTD_compressBound(). ZSTD_compressContinue() doesn't guarantee recover after a failed compression. @@ -347,9 +346,9 @@ ZSTDLIB_API size_t ZSTD_compressContinueThenEnd(ZSTD_CCtx* cctx, void* dst, size - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. In which case, it will "discard" the relevant memory section from its history. - Finish a frame with ZSTD_compressEnd(), which will write the epilogue, - or ZSTD_compressContinueThenEnd(), which will write the last block. - Without last block / epilogue mark, frames will be considered unfinished (broken) by decoders. + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame, + Without last block mark, frames will be considered unfinished (broken) by decoders. You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame. */ diff --git a/programs/fullbench.c b/programs/fullbench.c index 53041db77..f6852f6d8 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -173,12 +173,9 @@ static size_t local_ZBUFF_decompress(void* dst, size_t dstCapacity, void* buff2, static ZSTD_CCtx* g_zcc = NULL; size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) { - size_t compressedSize; (void)buff2; ZSTD_compressBegin(g_zcc, 1); - compressedSize = ZSTD_compressContinue(g_zcc, dst, dstCapacity, src, srcSize); - compressedSize += ZSTD_compressEnd(g_zcc, ((char*)dst)+compressedSize, dstCapacity-compressedSize); - return compressedSize; + return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize); } size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 33cfbcb2d..cb31dc431 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -186,11 +186,9 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++); cSize = 0; - CHECKPLUS(r, ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize), + CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize), (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), cSize += r); - CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize), - cSize += r); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); @@ -204,11 +202,9 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++); { size_t const cSizeOrig = cSize; cSize = 0; - CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize), + CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize), (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), cSize += r); - CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize), - cSize += r); if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */ } DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); @@ -696,7 +692,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD totalTestSize += segmentSize; } } - { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); + { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); cSize += flushResult; }