From: Yann Collet Date: Thu, 19 Dec 2024 15:26:38 +0000 (-0800) Subject: ZSTD_compressSequencesAndLiterals requires srcSize as parameter X-Git-Tag: v1.5.7^2~48^2~11 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0a54f6f288bab194e1c4edcd03cedc8d084c2d6f;p=thirdparty%2Fzstd.git ZSTD_compressSequencesAndLiterals requires srcSize as parameter this makes it possible to adjust windowSize to its tightest. --- diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 2c4c4b0e8..a50dd3c5d 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1421,7 +1421,7 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx, ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const ZSTD_Sequence* inSeqs, size_t nbSequences, - const void* literals, size_t litSize); + const void* literals, size_t litSize, size_t srcSize);

This is a variant of ZSTD_compressSequences() which, instead of receiving (src,srcSize) as input parameter, receives (literals,litSize), aka all the literals, already extracted and laid out into a single continuous buffer. diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 867e7fbf1..655adf34b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -7291,7 +7291,7 @@ size_t ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* literals, size_t litSize) + const void* literals, size_t litSize, size_t srcSize) { BYTE* op = (BYTE*)dst; size_t cSize = 0; @@ -7299,7 +7299,7 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, /* Transparent initialization stage, same as compressStream2() */ DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals (dstCapacity=%zu)", dstCapacity); assert(cctx != NULL); - FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_continue, 0), "CCtx initialization failed"); + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); if (cctx->appliedParams.blockDelimiters == ZSTD_sf_noBlockDelimiters) { RETURN_ERROR(frameParameter_unsupported, "This mode is only compatible with explicit delimiters"); @@ -7310,7 +7310,7 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, /* Begin writing output, starting with frame header */ { size_t const frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, - &cctx->appliedParams, ZSTD_CONTENTSIZE_UNKNOWN, cctx->dictID); + &cctx->appliedParams, srcSize, cctx->dictID); op += frameHeaderSize; assert(frameHeaderSize <= dstCapacity); dstCapacity -= frameHeaderSize; diff --git a/lib/zstd.h b/lib/zstd.h index 3fb02619c..68e78b3ca 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1675,7 +1675,6 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx, * but it also features the following limitations: * - Only supports explicit delimiter mode * - Not compatible with frame checksum, which must disabled - * - Does not write the content size in frame header * - If any block is incompressible, will fail and return an error * - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error. * - the buffer @literals must be larger than @litSize by at least 8 bytes. @@ -1685,7 +1684,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const ZSTD_Sequence* inSeqs, size_t nbSequences, - const void* literals, size_t litSize); + const void* literals, size_t litSize, size_t srcSize); /*! ZSTD_writeSkippableFrame() : diff --git a/tests/fullbench.c b/tests/fullbench.c index 9505110ec..8601af769 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -608,8 +608,8 @@ static PrepResult prepSequencesAndLiterals(const void* src, size_t srcSize, int static size_t local_compressSequencesAndLiterals(const void* input, size_t inputSize, - void* dst, size_t dstCapacity, - void* payload) + void* dst, size_t dstCapacity, + void* payload) { const char* ip = input; size_t srcSize = MEM_read32(ip); @@ -623,9 +623,9 @@ local_compressSequencesAndLiterals(const void* input, size_t inputSize, ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_repcodeResolution, ZSTD_ps_enable); #endif assert(12 + nbSeqs * sizeof(ZSTD_Sequence) + nbLiterals == inputSize); (void)inputSize; - (void)payload; (void)srcSize; + (void)payload; - return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals); + return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals, srcSize); } static PrepResult prepConvertSequences(const void* src, size_t srcSize, int cLevel) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 09572e909..4901376ff 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -3909,21 +3909,21 @@ static int basicUnitTests(U32 const seed, double compressibility) FUZ_transferLiterals(litBuffer, decompressSize, CNBuffer, srcSize, seqs, nbSeqs); /* not enough literals: must fail */ - compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1); + compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, srcSize); if (!ZSTD_isError(compressedSize)) { DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: not enough literals provided\n"); goto _output_error; } /* too many literals: must fail */ - compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1); + compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, srcSize); if (!ZSTD_isError(compressedSize)) { DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: too many literals provided\n"); goto _output_error; } /* correct amount of literals: should compress successfully */ - compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize); + compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, srcSize); if (ZSTD_isError(compressedSize)) { DISPLAY("Error in ZSTD_compressSequencesAndLiterals()\n"); goto _output_error;