From: Yann Collet Date: Sun, 2 Jan 2022 07:15:34 +0000 (-0800) Subject: make stableSrc compatible with regular streaming API X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2f4c9f96354096e4fb8d2da00845e4765e00c015;p=thirdparty%2Fzstd.git make stableSrc compatible with regular streaming API including flushStream(). Now the only condition is for `input.size` to continuously grow. --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8a52876ab..84009414a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -5336,7 +5336,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, U32 someMoreWork = 1; /* check expectations */ - DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i", (int)flushMode); if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { assert(zcs->inBuff != NULL); assert(zcs->inBuffSize > 0); @@ -5422,9 +5422,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, if (!lastBlock) assert(zcs->inBuffTarget <= zcs->inBuffSize); zcs->inToCompress = zcs->inBuffPos; - } else { + } else { /* !inputBuffered, hence ZSTD_bm_stable */ unsigned const lastBlock = (ip + iSize == iend); - assert(flushMode == ZSTD_e_end /* Already validated */); cSize = lastBlock ? ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) : ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize); @@ -5531,11 +5530,10 @@ static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, { if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { ZSTD_inBuffer const expect = cctx->expectedInBuffer; - if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size) + if (expect.src != input->src || expect.pos != input->pos) RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!"); - if (endOp != ZSTD_e_end) - RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!"); } + (void)endOp; if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { size_t const outBufferSize = output->size - output->pos; if (cctx->expectedOutBufferSize != outBufferSize) @@ -5648,7 +5646,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, && (endOp == ZSTD_e_continue) /* more to come */ && (input->pos < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */ cctx->expectedInBuffer = *input; - return (ZSTD_BLOCKSIZE_MAX - input->pos); /* don't do anything : allows lazy compression parameters adaptation */ + return (ZSTD_BLOCKSIZE_MAX - input->pos); /* don't do anything : allows lazy adaptation of compression parameters */ } FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ @@ -6165,7 +6163,10 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapaci * @return : amount of data remaining to flush */ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - ZSTD_inBuffer input = { NULL, 0, 0 }; + ZSTD_inBuffer const nullInput = { NULL, 0, 0 }; + int const stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + ZSTD_inBuffer input = stableInput ? zcs->expectedInBuffer : nullInput; + input.size = input.pos; /* do not ingest more input during flush */ return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); } diff --git a/lib/zstd.h b/lib/zstd.h index 003db5513..8bd85e921 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1829,13 +1829,16 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo * Experimental parameter. * Default is 0 == disabled. Set to 1 to enable. * - * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the compressor, and - * compression will fail if it ever changes. This means the only flush - * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end - * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) - * MUST not be modified during compression or you will get data corruption. + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. * * When this flag is enabled zstd won't allocate an input window buffer, * because the user guarantees it can reference the ZSTD_inBuffer until @@ -1843,18 +1846,15 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also * avoid the memcpy() from the input buffer to the input window buffer. * - * NOTE: ZSTD_compressStream2() will error if ZSTD_e_flush is used. - * That means this flag cannot be used with ZSTD_flushStream(). - * * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, compression WILL fail if you violate the preconditions. + * memory. However, compression WILL fail if conditions are not respected. * * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST - * not be modified during compression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_inBuffer to find + * not be modified during compression or it sill result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find * matches. Normally zstd maintains its own window buffer for this purpose, - * but passing this flag tells zstd to use the user provided buffer. + * but passing this flag tells zstd to rely on user provided buffer instead. */ #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 72fd72ea3..66e017c3d 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -613,7 +613,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); } } - /* Complex context re-use scenario */ + /* Compression state re-use scenario */ DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++); ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); @@ -634,8 +634,7 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ DISPLAYLEVEL(5, "end1 "); - { size_t const r = ZSTD_endStream(zc, &outBuff); - if (r != 0) goto _output_error; } /* error, or some data not flushed */ + if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */ } /* use 2 */ { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */ @@ -653,8 +652,7 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ DISPLAYLEVEL(5, "end2 "); - { size_t const r = ZSTD_endStream(zc, &outBuff); - if (r != 0) goto _output_error; } /* error, or some data not flushed */ + if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */ } DISPLAYLEVEL(3, "OK \n"); @@ -786,16 +784,18 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test"); CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize)); CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx)); - { ZSTD_CCtx* cctx2 = ZSTD_createCCtx(); + { ZSTD_CCtx* const cctx2 = ZSTD_createCCtx(); + assert(cctx2 != NULL); in.pos = out.pos = 0; CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue)); CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished"); CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2)); ZSTD_freeCCtx(cctx2); } - { ZSTD_CCtx* cctx3 = ZSTD_createCCtx(); + { ZSTD_CCtx* const cctx3 = ZSTD_createCCtx(); ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0); size_t cSize3; + assert(cctx3 != NULL); params.fParams.checksumFlag = 1; cSize3 = ZSTD_compress_advanced(cctx3, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params); CHECK_Z(cSize3); @@ -808,8 +808,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++); - { - int stableInBuffer; + { int stableInBuffer; int stableOutBuffer; CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer)); CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer)); @@ -870,21 +869,24 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer with continue and flush : ", testNb++); + /* stableSrc + streaming */ + DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream, flushStream and endStream : ", testNb++); + CHECK_Z( ZSTD_initCStream(cctx, 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) ); in.src = CNBuffer; - in.size = CNBufferSize; + in.size = 100; in.pos = 0; - out.pos = 0; - out.size = compressedBufferSize; - CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only)); - { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); - CHECK(!ZSTD_isError(ret), "Must error"); - CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error"); - } - CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only)); - { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush); - CHECK(!ZSTD_isError(ret), "Must error"); - CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error"); + { ZSTD_outBuffer outBuf; + outBuf.dst = (char*)(compressedBuffer)+cSize; + outBuf.size = ZSTD_compressBound(500); + outBuf.pos = 0; + CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &in) ); + in.size = 200; + CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &in) ); + CHECK_Z( ZSTD_flushStream(cctx, &outBuf) ); + in.size = 300; + CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &in) ); + if (ZSTD_endStream(cctx, &outBuf) != 0) goto _output_error; /* error, or some data not flushed */ } DISPLAYLEVEL(3, "OK \n"); @@ -902,6 +904,7 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1)); in.pos = out.pos = 0; in.size = MIN(CNBufferSize, 10); + out.size = compressedBufferSize; CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); in.pos = 0; in.size = CNBufferSize - in.size;