*/
static size_t
ZSTD_transferSequencesOnly_wBlockDelim(ZSTD_CCtx* cctx,
- ZSTD_SequencePosition* seqPos,
+ ZSTD_SequencePosition* seqPos, size_t* litConsumedPtr,
const ZSTD_Sequence* const inSeqs, size_t nbSequences,
size_t blockSize,
ZSTD_ParamSwitch_e externalRepSearch)
Repcodes_t updatedRepcodes;
U32 dictSize;
size_t startPosInSrc = seqPos->posInSrc;
+ size_t litConsumed = 0;
DEBUGLOG(5, "ZSTD_transferSequencesOnly_wBlockDelim (blockSize = %zu)", blockSize);
RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
"Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
ZSTD_storeSeqOnly(&cctx->seqStore, litLength, offBase, matchLength);
+ litConsumed += litLength;
}
/* last sequence (only literals) */
- seqPos->posInSrc += inSeqs[idx].litLength;
+ { size_t const lastLitLength = inSeqs[idx].litLength;
+ seqPos->posInSrc += lastLitLength;
+ cctx->seqStore.lit += lastLitLength; /* register proper length */
+ litConsumed += lastLitLength;
+ }
/* blockSize must be exactly correct (checked before calling this function) */
assert((seqPos->posInSrc - startPosInSrc) == blockSize); (void)startPosInSrc;
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t));
seqPos->idx = idx+1;
+ *litConsumedPtr = litConsumed;
return blockSize;
}
}
while (remaining) {
- size_t compressedSeqsSize;
- size_t cBlockSize;
+ size_t compressedSeqsSize, cBlockSize, litConsumed;
size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters,
cctx->blockSize, remaining,
inSeqs, nbSequences, seqPos);
U32 const lastBlock = (blockSize == remaining);
FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size");
+ RETURN_ERROR_IF(!lastBlock, GENERIC, "Only supports single block");
assert(blockSize <= remaining);
ZSTD_resetSeqStore(&cctx->seqStore);
blockSize = ZSTD_transferSequencesOnly_wBlockDelim(cctx,
- &seqPos,
+ &seqPos, &litConsumed,
inSeqs, nbSequences,
blockSize,
cctx->appliedParams.searchForExternalRepcodes);
+ RETURN_ERROR_IF(blockSize != remaining, GENERIC, "Must consume the entire block");
+ RETURN_ERROR_IF(litConsumed != litSize, GENERIC, "Must consume the exact amount of literals provided");
FORWARD_IF_ERROR(blockSize, "Bad sequence copy");
/* Note: when blockSize is very small, other variant send it uncompressed.
* This can be useful if the process generating the sequences also happens to generate the buffer of literals,
* thus skipping an extraction + caching stage.
* It's essentially a speed optimization when the right conditions are met,
- * but it also is restricted by the following limitations:
+ * but it also features the following limitations:
* - Only supports explicit delimiter mode
+ * - Supports 1 block only (max input 128 KB)
* - Not compatible with frame checksum, which must disabled
- * - Can fail when unable to compress sufficiently
+ * - Can fail (return an error) when input data cannot be compress sufficiently
* Also, to be valid, @litSize must be equal to the sum of all @.litLength fields in @inSeqs.
* @return : final compressed size, or a ZSTD error code.
*/
ZSTDLIB_STATIC_API size_t
ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
- const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+ const ZSTD_Sequence* inSeqs, size_t nbSequences,
const void* literals, size_t litSize, size_t srcSize);
#include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH64 */
-#include "util.h"
#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
/* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
}
}
+static size_t FUZ_getLitSize(const ZSTD_Sequence* seqs, size_t nbSeqs)
+{
+ size_t n, litSize = 0;
+ assert(seqs != NULL);
+ for (n=0; n<nbSeqs; n++) {
+ litSize += seqs[n].litLength;
+ }
+ return litSize;
+}
+
+static void
+FUZ_transferLiterals(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_Sequence* seqs, size_t nbSeqs)
+{
+ size_t n;
+ const char* ip = (const char*)src;
+ char* op = (char*)dst;
+ size_t const litSize = FUZ_getLitSize(seqs, nbSeqs);
+ assert(litSize <= dstCapacity);
+ for (n=0; n<nbSeqs; n++) {
+ size_t const ll = seqs[n].litLength;
+ memcpy(op, ip, ll);
+ op += ll;
+ ip += ll + seqs[n].matchLength;
+ }
+ assert((size_t)(ip - (const char*)src) == srcSize);
+}
+
#ifdef ZSTD_MULTITHREAD
typedef struct {
if (seqs == NULL) goto _output_error;
assert(cctx != NULL);
- /* Populate src with random data */
+ /* Populate src with compressible random data */
RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
/* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */
}
DISPLAYLEVEL(3, "OK \n");
+ DISPLAYLEVEL(3, "test%3i : ZSTD_compressSequencesAndLiterals : ", testNb++);
+ {
+ const size_t srcSize = 100 KB;
+ const BYTE* const src = (BYTE*)CNBuffer;
+ BYTE* const dst = (BYTE*)compressedBuffer;
+ const size_t dstCapacity = ZSTD_compressBound(srcSize);
+ const size_t decompressSize = srcSize;
+ char* const decompressBuffer = (char*)malloc(decompressSize);
+ char* const litBuffer = (char*)malloc(decompressSize);
+ size_t compressedSize;
+
+ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+ ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
+ size_t nbSeqs;
+
+ if (litBuffer == NULL) goto _output_error;
+ if (decompressBuffer == NULL) goto _output_error;
+ if (seqs == NULL) goto _output_error;
+ assert(cctx != NULL);
+
+ /* Populate src with compressible random data */
+ RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
+
+ /* Roundtrip Test using the AndLiterals() variant */
+ nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
+ { size_t const litSize = FUZ_getLitSize(seqs, nbSeqs);
+ FUZ_transferLiterals(litBuffer, decompressSize, CNBuffer, srcSize, seqs, nbSeqs);
+
+ /* not enough literals: must fail */
+ compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, srcSize);
+ if (!ZSTD_isError(compressedSize)) {
+ DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: not enough literals provided\n");
+ goto _output_error;
+ }
+
+ /* too many literals: must fail */
+ compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, srcSize);
+ if (!ZSTD_isError(compressedSize)) {
+ DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: too many literals provided\n");
+ goto _output_error;
+ }
+
+ /* correct amount of literals: should compress successfully */
+ compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, srcSize);
+ if (ZSTD_isError(compressedSize)) {
+ DISPLAY("Error in ZSTD_compressSequencesAndLiterals()\n");
+ goto _output_error;
+ }
+ }
+ { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
+ if (ZSTD_isError(dSize)) {
+ DISPLAY("Error during decompression of frame produced by ZSTD_compressSequencesAndLiterals()\n");
+ goto _output_error;
+ }
+ if (dSize != srcSize) {
+ DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() has different size\n");
+ goto _output_error;
+ }
+ if (memcmp(decompressBuffer, src, srcSize)) {
+ DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() produces a different content (of same size)\n");
+ goto _output_error;
+ }
+ }
+
+ ZSTD_freeCCtx(cctx);
+ free(litBuffer);
+ free(decompressBuffer);
+ free(seqs);
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
/* Multiple blocks of zeros test */
#define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
projects / thirdparty / zstd.git / commitdiff
