****************************************************************/
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
{
- size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 1 + 1; /* last +1 : written by U16 */
+ size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
}
}
-size_t FSE_writeNCount (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
+size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */
if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */
- if (headerBufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
- return FSE_writeNCount_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
+ if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
- return FSE_writeNCount_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
}
}
else
{
- ip = iend - 4;
bitCount -= (int)(8 * (iend - 4 - ip));
- }
+ ip = iend - 4;
+ }
bitStream = FSE_readLE32(ip) >> (bitCount & 31);
}
}
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{
U32 tableLog = maxTableLog;
+ U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
+ U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
+ U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
- if ((FSE_highbit32((U32)(srcSize - 1)) - 2) < tableLog) tableLog = FSE_highbit32((U32)(srcSize - 1)) - 2; /* Accuracy can be reduced */
- if ((FSE_highbit32(maxSymbolValue)+2) > tableLog) tableLog = FSE_highbit32(maxSymbolValue)+2; /* Need a minimum to safely represent all symbol values */
+ if (minBitsSrc < tableLog + 3) tableLog = minBitsSrc-3; /* Accuracy can be reduced */
+ if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
return tableLog;
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */
if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */
- if ((1U<<tableLog) <= maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; /* Too small tableLog, compression potentially impossible */
+
//if ((1U<<tableLog) <= maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; /* Too small tableLog, compression potentially impossible */
{
U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
{
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
+ return FSE_DStream_tooFar;
+
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
{
bitD->ptr -= bitD->bitsConsumed >> 3;
if (bitD->ptr == bitD->start)
{
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_endOfBuffer;
- if (bitD->bitsConsumed == sizeof(bitD->bitContainer)*8) return FSE_DStream_completed;
- return FSE_DStream_tooFar;
+ return FSE_DStream_completed;
}
{
U32 nbBytes = bitD->bitsConsumed >> 3;
U32 result = FSE_DStream_unfinished;
if (bitD->ptr - nbBytes < bitD->start)
{
- nbBytes = (U32)(bitD->ptr - bitD->start); /* note : necessarily ptr > start */
+ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
result = FSE_DStream_endOfBuffer;
}
bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes*8;
- bitD->bitContainer = FSE_readLEST(bitD->ptr); /* note : necessarily srcSize > sizeof(bitD) */
+ bitD->bitContainer = FSE_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
return result;
}
}
FSE_CStream_t bitC;
/* init */
- if (dstSize < 8) return 0; /* need a minimum for jumpTable and first symbols */
+ if (dstSize < 8) return 0;
op += 6; /* jump Table -- could be optimized by delta / deviation */
errorCode = FSE_initCStream(&bitC, op, oend-op);
if (FSE_isError(errorCode)) return 0;
U32 LLlog, Offlog, MLlog;
size_t dumpsLength;
+ /* check */
+ if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
+
/* SeqHead */
*nbSeq = ZSTD_readLE16(ip); ip+=2;
LLtype = *ip >> 6;
*dumpsPtr = ip;
ip += dumpsLength;
+ /* check */
+ if (ip > iend-1) return (size_t)-ZSTD_ERROR_SrcSize;
+
/* sequences */
{
S16 norm[MaxML+1]; /* assumption : MaxML >= MaxLL and MaxOff */
max = MaxLL;
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+ if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize;
FSE_buildDTable(DTableLL, norm, max, LLlog);
}
max = MaxOff;
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+ if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize;
FSE_buildDTable(DTableOffb, norm, max, Offlog);
}
max = MaxML;
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+ if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize;
FSE_buildDTable(DTableML, norm, max, MLlog);
}
}
-static size_t ZSTD_execSequence(BYTE* op, seq_t sequence, const BYTE** litPtr, BYTE* const oend)
+static size_t ZSTD_execSequence(BYTE* op,
+ seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ BYTE* const base, BYTE* const oend)
{
static const int dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; /* added */
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
/* check */
if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+ if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
/* copy Literals */
if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8))
size_t qutt = 12;
U64 saved[2];
+ /* check */
+ if (match < base) return (size_t)-ZSTD_ERROR_corruption;
+ if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
+
/* save beginning of literal sequence, in case of write overlap */
if (overlapRisk)
{
return endMatch-ostart;
}
+typedef struct ZSTD_Dctx_s
+{
+ U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
+ U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+ U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+ void* previousDstEnd;
+ void* base;
+ size_t expected;
+ blockType_t bType;
+ U32 phase;
+} dctx_t;
+
static size_t ZSTD_decompressSequences(
void* ctx,
const void* seqStart, size_t seqSize,
const BYTE* litStart, size_t litSize)
{
+ dctx_t* dctx = (dctx_t*)ctx;
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE* const)dst;
const BYTE* const litEnd = litStart + litSize;
int nbSeq;
const BYTE* dumps;
- FSE_DTable* DTableML = (FSE_DTable*)ctx;
- FSE_DTable* DTableLL = DTableML + FSE_DTABLE_SIZE_U32(MLFSELog);
- FSE_DTable* DTableOffb = DTableLL + FSE_DTABLE_SIZE_U32(LLFSELog);
+ U32* DTableLL = dctx->LLTable;
+ U32* DTableML = dctx->MLTable;
+ U32* DTableOffb = dctx->OffTable;
+ BYTE* const base = (BYTE*) (dctx->base);
/* Build Decoding Tables */
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps,
size_t oneSeqSize;
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, oend);
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
}
-#if 0
-FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize,
- const void* src, size_t srcSize)
-{
- const BYTE* ip = (const BYTE*)src;
- const BYTE* const iend = ip + srcSize;
- BYTE* const ostart = (BYTE* const)dst;
- BYTE* op = ostart;
- BYTE* const oend = ostart + maxDstSize;
- size_t errorCode;
- size_t lastLLSize;
- const BYTE* dumps;
- const BYTE* litPtr;
- const BYTE* litEnd;
- const int dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; /* added */
- const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
- FSE_DTable* DTableML = (FSE_DTable*)ctx;
- FSE_DTable* DTableLL = DTableML + FSE_DTABLE_SIZE_U32(MLFSELog);
- FSE_DTable* DTableOffb = DTableLL + FSE_DTABLE_SIZE_U32(LLFSELog);
-
- /* blockType == blockCompressed, srcSize is trusted */
-
- /* Decode literals sub-block */
- errorCode = ZSTD_decodeLiteralsBlock(ctx, dst, maxDstSize, &litPtr, src, srcSize);
- if (ZSTD_isError(errorCode)) return errorCode;
- ip += errorCode;
-
- /* Build Decoding Tables */
- errorCode = ZSTD_decodeSeqHeaders(&lastLLSize, &dumps,
- DTableLL, DTableML, DTableOffb,
- ip, iend-ip);
- if (ZSTD_isError(errorCode)) return errorCode;
- /* end pos */
- if ((litPtr>=ostart) && (litPtr<=oend)) /* decoded literals are into dst buffer */
- litEnd = oend - lastLLSize;
- else
- litEnd = ip - lastLLSize;
- ip += errorCode;
-
- /* LZ Sequences */
- {
- FSE_DStream_t DStream;
- FSE_DState_t stateLL, stateOffb, stateML;
- size_t prevOffset = 0, offset = 0;
-
- FSE_initDStream(&DStream, ip, iend-ip);
- FSE_initDState(&stateLL, &DStream, DTableLL);
- FSE_initDState(&stateOffb, &DStream, DTableOffb);
- FSE_initDState(&stateML, &DStream, DTableML);
-
- while (FSE_reloadDStream(&DStream)<2)
- {
- U32 nbBits, offsetCode;
- const BYTE* match;
- size_t litLength;
- size_t matchLength;
- size_t newOffset;
-
-_another_round:
-
- /* Literals */
- litLength = FSE_decodeSymbol(&stateLL, &DStream);
- if (litLength) prevOffset = offset;
- if (litLength == MaxLL)
- {
- BYTE add = *dumps++;
- if (add < 255) litLength += add;
- else
- {
- litLength = ZSTD_readLE32(dumps) & 0xFFFFFF;
- dumps += 3;
- }
- }
- if (((size_t)(litPtr - op) < 8) || ((size_t)(oend-(litPtr+litLength)) < 8))
- memmove(op, litPtr, litLength); /* overwrite risk */
- else
- ZSTD_wildcopy(op, litPtr, litLength);
- op += litLength;
- litPtr += litLength;
-
- /* Offset */
- offsetCode = FSE_decodeSymbol(&stateOffb, &DStream);
- if (ZSTD_32bits()) FSE_reloadDStream(&DStream);
- nbBits = offsetCode - 1;
- if (offsetCode==0) nbBits = 0; /* cmove */
- newOffset = FSE_readBits(&DStream, nbBits);
- if (ZSTD_32bits()) FSE_reloadDStream(&DStream);
- newOffset += (size_t)1 << nbBits;
- if (offsetCode==0) newOffset = prevOffset;
- match = op - newOffset;
- prevOffset = offset;
- offset = newOffset;
-
- /* MatchLength */
- matchLength = FSE_decodeSymbol(&stateML, &DStream);
- if (matchLength == MaxML)
- {
- BYTE add = *dumps++;
- if (add < 255) matchLength += add;
- else
- {
- matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
- dumps += 3;
- }
- }
- matchLength += MINMATCH;
-
- /* copy Match */
- {
- BYTE* const endMatch = op + matchLength;
- size_t qutt=12;
- U64 saved[2];
- const U32 overlapRisk = (((size_t)(litPtr - endMatch)) < 12);
-
- /* save beginning of literal sequence, in case of write overlap */
- if (overlapRisk)
- {
- if ((endMatch + qutt) > oend) qutt = oend-endMatch;
- memcpy(saved, endMatch, qutt);
- }
-
- if (offset < 8)
- {
- const int dec64 = dec64table[offset];
- op[0] = match[0];
- op[1] = match[1];
- op[2] = match[2];
- op[3] = match[3];
- match += dec32table[offset];
- ZSTD_copy4(op+4, match);
- match -= dec64;
- } else { ZSTD_copy8(op, match); }
-
- if (endMatch > oend-12)
- {
- if (op < oend-16)
- {
- ZSTD_wildcopy(op+8, match+8, (oend-8) - (op+8));
- match += (oend-8) - op;
- op = oend-8;
- }
- while (op<endMatch) *op++ = *match++;
- }
- else
- ZSTD_wildcopy(op+8, match+8, matchLength-8); /* works even if matchLength < 8 */
-
- op = endMatch;
-
- /* restore, in case of overlap */
- if (overlapRisk)
- memcpy(endMatch, saved, qutt);
- }
- }
-
- /* check if reached exact end */
- if (FSE_reloadDStream(&DStream) > 2) return (size_t)-ZSTD_ERROR_GENERIC; /* requested too much : data is corrupted */
- if (!FSE_endOfDState(&stateLL) && !FSE_endOfDState(&stateML) && !FSE_endOfDState(&stateOffb)) goto _another_round; /* some ultra-compressible sequence remain ! */
- if (litPtr != litEnd) goto _another_round; /* literals not entirely spent */
-
- /* last literal segment */
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
- op += lastLLSize;
- }
-
- return op-ostart;
-}
-#endif
-
-
static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
return op-ostart;
}
-
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
- U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)];
- return ZSTD_decompressDCtx(ctx, dst, maxDstSize, src, srcSize);
+ dctx_t ctx;
+ ctx.base = dst;
+ return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
}
* Streaming Decompression API
*******************************/
-typedef struct ZSTD_Dctx_s
-{
- U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)];
- size_t expected;
- blockType_t bType;
- U32 phase;
-} dctx_t;
-
-
ZSTD_Dctx* ZSTD_createDCtx(void)
{
ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
if (dctx==NULL) return NULL;
dctx->expected = ZSTD_frameHeaderSize;
dctx->phase = 0;
+ dctx->previousDstEnd = NULL;
+ dctx->base = NULL;
return dctx;
}
/* Sanity check */
if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize;
+ if (dst != ctx->previousDstEnd) /* not contiguous */
+ ctx->base = dst;
/* Decompress : frame header */
if (ctx->phase == 0)
}
ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize;
+ ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
return rSize;
}
}
-/*
-static unsigned FUZ_highbit(U32 v32)
+static unsigned FUZ_highbit32(U32 v32)
{
unsigned nbBits = 0;
if (v32==0) return 0;
}
return nbBits;
}
-*/
static int basicUnitTests(U32 seed, double compressibility)
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize);
CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
}
+
+ /* noisy src decompression test */
+ if (cSize > 6)
+ {
+ const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4));
+ size_t pos = 4; /* preserve magic number (too easy to detect) */
+ U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
+ size_t mask = (1<<nbBits) - 1;
+ size_t skipLength = FUZ_rand(&lseed) & mask;
+ pos += skipLength;
+
+ while (pos < cSize)
+ {
+ /* add noise */
+ size_t noiseStart, noiseLength;
+ nbBits = FUZ_rand(&lseed) % maxNbBits;
+ if (nbBits>0) nbBits--;
+ mask = (1<<nbBits) - 1;
+ noiseLength = (FUZ_rand(&lseed) & mask) + 1;
+ noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
+ memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
+ pos += noiseLength;
+
+ /* keep some original src */
+ nbBits = FUZ_rand(&lseed) % maxNbBits;
+ mask = (1<<nbBits) - 1;
+ skipLength = FUZ_rand(&lseed) & mask;
+ pos += skipLength;
+ }
+
+ /* decompress noisy source */
+ {
+ const U32 endMark = 0xA9B1C3D6;
+ U32 endCheck;
+ size_t errorCode;
+ memcpy(dstBuffer+sampleSize, &endMark, 4);
+ errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
+ /* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */
+ CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize),
+ "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize);
+ memcpy(&endCheck, dstBuffer+sampleSize, 4);
+ CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
+ }
+ }
}
DISPLAY("\rAll fuzzer tests completed \n");
projects / thirdparty / zstd.git / commitdiff
