U32* hashTable;
U32* contentTable;
HUF_CElt* hufTable;
- U32 flagHufTable;
+ U32 flagStaticTables;
+ FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
+ FSE_CTable matchlengthCTable [FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
+ FSE_CTable litlengthCTable [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
};
if (params->windowLog < ZSTD_WINDOWLOG_MIN) params->windowLog = ZSTD_WINDOWLOG_MIN;
/* correct params, to use less memory */
- if ((params->srcSize > 0) && (params->srcSize < (1<<ZSTD_WINDOWLOG_MAX)))
- {
+ if ((params->srcSize > 0) && (params->srcSize < (1<<ZSTD_WINDOWLOG_MAX))) {
U32 srcLog = ZSTD_highbit((U32)(params->srcSize)-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog;
}
const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog)) * sizeof(U32);
const size_t neededSpace = tableSpace + (256*sizeof(U32)) + (3*blockSize);
- if (zc->workSpaceSize < neededSpace)
- {
+ if (zc->workSpaceSize < neededSpace) {
free(zc->workSpace);
zc->workSpace = malloc(neededSpace);
if (zc->workSpace == NULL) return ERROR(memory_allocation);
zc->contentTable = zc->hashTable + ((size_t)1 << params.hashLog);
zc->seqStore.buffer = zc->contentTable + ((size_t)1 << contentLog);
zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
- zc->flagHufTable = 0;
+ zc->flagStaticTables = 0;
zc->seqStore.buffer = (U32*)(zc->seqStore.buffer) + 256;
}
dstCCtx->dictLimit = srcCCtx->dictLimit;
dstCCtx->lowLimit = srcCCtx->lowLimit;
- dstCCtx->flagHufTable = srcCCtx->flagHufTable;
- if (dstCCtx->flagHufTable)
+ /* copy entropy tables */
+ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
+ if (dstCCtx->flagStaticTables) {
memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
+ memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable));
+ memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable));
+ memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
+ }
return 0;
}
U32* table32 = zc->hashTable;
U32 index;
- for (index=0 ; index < tableSpaceU32 ; index++)
- {
+ for (index=0 ; index < tableSpaceU32 ; index++) {
if (table32[index] < reducerValue) table32[index] = 0;
else table32[index] -= reducerValue;
}
1.2.1) Huff0 block, using sizes from header
See Huff0 format
- 1.2.2) Raw content
+ 1.2.2) Huff0 block, using prepared table
- 1.2.3) single byte
+ 1.2.3) Raw content
- 1.2.4) _usingDTable variant ?
+ 1.2.4) single byte
2) Sequences section
- TO DO
+
+ - Nb Sequences : 2 bytes, little endian
+ - Control Token : 1 byte (see below)
+ - Dumps Length : 1 or 2 bytes (depending on control token)
+ - Dumps : as stated by dumps length
+ - Literal Lengths FSE table (as needed depending on encoding method)
+ - Offset Codes FSE table (as needed depending on encoding method)
+ - Match Lengths FSE table (as needed depending on encoding method)
+
+ 2.1) Control Token
+ 8 bits, divided as :
+ 0-1 : dumpsLength
+ 2-3 : MatchLength, FSE encoding method
+ 4-5 : Offset Codes, FSE encoding method
+ 6-7 : Literal Lengths, FSE encoding method
+
+ FSE encoding method :
+ FSE_ENCODING_RAW : uncompressed; no header
+ FSE_ENCODING_RLE : single repeated value; header 1 byte
+ FSE_ENCODING_STATIC : use prepared table; no header
+ FSE_ENCODING_DYNAMIC : read NCount
*/
size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
BYTE* const ostart = (BYTE* const)dst;
U32 flSize = 1 + (srcSize>31) + (srcSize>4095);
- (void)maxDstSize; /* maxDstSize guaranteed to be >=4, hence large enough ? */
+ (void)maxDstSize; /* maxDstSize guaranteed to be >=4, hence large enough */
switch(flSize)
{
if (maxDstSize < 4) return ERROR(dstSize_tooSmall); /* not enough space for compression */
- if (zc->flagHufTable && (lhSize==3))
- {
+ if (zc->flagStaticTables && (lhSize==3)) {
hType = IS_PCH;
singleStream = 1;
clitSize = HUF_compress1X_usingCTable(ostart+lhSize, maxDstSize-lhSize, src, srcSize, zc->hufTable);
- }
- else
- {
+ } else {
clitSize = singleStream ? HUF_compress1X(ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12)
: HUF_compress2 (ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12);
}
U32 count[MaxSeq+1];
S16 norm[MaxSeq+1];
size_t mostFrequent;
- U32 max = 255;
- U32 tableLog = 11;
- U32 CTable_LitLength [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL )];
- U32 CTable_OffsetBits [FSE_CTABLE_SIZE_U32(OffFSELog,MaxOff)];
- U32 CTable_MatchLength[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML )];
+ U32 max;
+ FSE_CTable* CTable_LitLength = zc->litlengthCTable;
+ FSE_CTable* CTable_OffsetBits = zc->offcodeCTable;
+ FSE_CTable* CTable_MatchLength = zc->matchlengthCTable;
U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
const BYTE* const op_lit_start = seqStorePtr->litStart;
const BYTE* const llTable = seqStorePtr->litLengthStart;
{
size_t cSize;
size_t litSize = seqStorePtr->lit - op_lit_start;
- const size_t minLitSize = zc->flagHufTable ? 6 : LITERAL_NOENTROPY;
+ const size_t minLitSize = zc->flagStaticTables ? 6 : LITERAL_NOENTROPY;
if (litSize <= minLitSize)
cSize = ZSTD_noCompressLiterals(op, maxDstSize, op_lit_start, litSize);
/* dumps : contains too large lengths */
{
size_t dumpsLength = seqStorePtr->dumps - seqStorePtr->dumpsStart;
- if (dumpsLength < 512)
- {
+ if (dumpsLength < 512) {
op[0] = (BYTE)(dumpsLength >> 8);
op[1] = (BYTE)(dumpsLength);
op += 2;
- }
- else
- {
+ } else {
op[0] = 2;
op[1] = (BYTE)(dumpsLength>>8);
op[2] = (BYTE)(dumpsLength);
op += dumpsLength;
}
+#define MIN_SEQ_FOR_DYNAMIC_FSE 64
+#define MAX_SEQ_FOR_STATIC_FSE 1000
+
/* CTable for Literal Lengths */
max = MaxLL;
mostFrequent = FSE_countFast(count, &max, seqStorePtr->litLengthStart, nbSeq);
- if ((mostFrequent == nbSeq) && (nbSeq > 2))
- {
+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = *(seqStorePtr->litLengthStart);
FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
- LLtype = bt_rle;
- }
- else if ((nbSeq < 64) || (mostFrequent < (nbSeq >> (LLbits-1))))
- {
+ LLtype = FSE_ENCODING_RLE;
+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+ LLtype = FSE_ENCODING_STATIC;
+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LLbits-1)))) {
FSE_buildCTable_raw(CTable_LitLength, LLbits);
- LLtype = bt_raw;
- }
- else
- {
+ LLtype = FSE_ENCODING_RAW;
+ } else {
size_t NCountSize;
- tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
+ U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return ERROR(GENERIC);
op += NCountSize;
FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
- LLtype = bt_compressed;
+ LLtype = FSE_ENCODING_DYNAMIC;
}
- /* CTable for Offsets codes */
- {
- /* create Offset codes */
- size_t i;
- max = MaxOff;
- for (i=0; i<nbSeq; i++)
- {
+ /* CTable for Offset codes */
+ { /* create Offset codes */
+ size_t i; for (i=0; i<nbSeq; i++) {
offCodeTable[i] = (BYTE)ZSTD_highbit(offsetTable[i]) + 1;
if (offsetTable[i]==0) offCodeTable[i]=0;
}
- mostFrequent = FSE_countFast(count, &max, offCodeTable, nbSeq);
}
- if ((mostFrequent == nbSeq) && (nbSeq > 2))
- {
+ max = MaxOff;
+ mostFrequent = FSE_countFast(count, &max, offCodeTable, nbSeq);
+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = *offCodeTable;
FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
- Offtype = bt_rle;
- }
- else if ((nbSeq < 64) || (mostFrequent < (nbSeq >> (Offbits-1))))
- {
+ Offtype = FSE_ENCODING_RLE;
+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+ Offtype = FSE_ENCODING_STATIC;
+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (Offbits-1)))) {
FSE_buildCTable_raw(CTable_OffsetBits, Offbits);
- Offtype = bt_raw;
- }
- else
- {
+ Offtype = FSE_ENCODING_RAW;
+ } else {
size_t NCountSize;
- tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
+ U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return ERROR(GENERIC);
op += NCountSize;
FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
- Offtype = bt_compressed;
+ Offtype = FSE_ENCODING_DYNAMIC;
}
/* CTable for MatchLengths */
max = MaxML;
mostFrequent = FSE_countFast(count, &max, seqStorePtr->matchLengthStart, nbSeq);
- if ((mostFrequent == nbSeq) && (nbSeq > 2))
- {
+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = *seqStorePtr->matchLengthStart;
FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
- MLtype = bt_rle;
- }
- else if ((nbSeq < 64) || (mostFrequent < (nbSeq >> (MLbits-1))))
- {
+ MLtype = FSE_ENCODING_RLE;
+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+ MLtype = FSE_ENCODING_STATIC;
+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (MLbits-1)))) {
FSE_buildCTable_raw(CTable_MatchLength, MLbits);
- MLtype = bt_raw;
- }
- else
- {
+ MLtype = FSE_ENCODING_RAW;
+ } else {
size_t NCountSize;
- tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
+ U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return ERROR(GENERIC);
op += NCountSize;
FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
- MLtype = bt_compressed;
+ MLtype = FSE_ENCODING_DYNAMIC;
}
seqHead[0] += (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
+ zc->flagStaticTables = 0;
/* Encoding Sequences */
{
FSE_initCState(&stateOffsetBits, CTable_OffsetBits);
FSE_initCState(&stateLitLength, CTable_LitLength);
- for (i=(int)nbSeq-1; i>=0; i--)
- {
- BYTE matchLength = mlTable[i];
+ for (i=(int)nbSeq-1; i>=0; i--) {
+ BYTE mlCode = mlTable[i];
U32 offset = offsetTable[i];
BYTE offCode = offCodeTable[i]; /* 32b*/ /* 64b*/
U32 nbBits = (offCode-1) * (!!offCode);
BYTE litLength = llTable[i]; /* (7)*/ /* (7)*/
- FSE_encodeSymbol(&blockStream, &stateMatchLength, matchLength); /* 17 */ /* 17 */
+ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 17 */ /* 17 */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
BIT_addBits(&blockStream, offset, nbBits); /* 31 */ /* 42 */ /* 24 bits max in 32-bits mode */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
#if 0
static const BYTE* g_start = NULL;
if (g_start==NULL) g_start = literals;
- if (literals - g_start == 8695)
+ //if (literals - g_start == 8695)
printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n",
(U32)(literals - g_start), (U32)litLength, (U32)matchCode+4, (U32)offsetCode);
#endif
seqStorePtr->lit += litLength;
/* literal Length */
- if (litLength >= MaxLL)
- {
+ if (litLength >= MaxLL) {
*(seqStorePtr->litLength++) = MaxLL;
- if (litLength<255 + MaxLL)
+ if (litLength<255 + MaxLL) {
*(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
- else
- {
+ } else {
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
- }
- }
+ } }
else *(seqStorePtr->litLength++) = (BYTE)litLength;
/* match offset */
*(seqStorePtr->offset++) = (U32)offsetCode;
/* match Length */
- if (matchCode >= MaxML)
- {
+ if (matchCode >= MaxML) {
*(seqStorePtr->matchLength++) = MaxML;
- if (matchCode < 255+MaxML)
+ if (matchCode < 255+MaxML) {
*(seqStorePtr->dumps++) = (BYTE)(matchCode - MaxML);
- else
- {
+ } else {
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)matchCode); seqStorePtr->dumps += 3;
- }
- }
+ } }
else *(seqStorePtr->matchLength++) = (BYTE)matchCode;
}
{
const BYTE* const pStart = pIn;
- while ((pIn<pInLimit-(sizeof(size_t)-1)))
- {
+ while ((pIn<pInLimit-(sizeof(size_t)-1))) {
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
const BYTE* ip = base + zc->nextToUpdate;
const BYTE* const iend = (const BYTE*) end;
- while(ip <= iend)
- {
+ while(ip <= iend) {
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
ip += FILLHASHSTEP;
}
/* init */
ZSTD_resetSeqStore(seqStorePtr);
- if (ip < lowest+4)
- {
+ if (ip < lowest+4) {
hashTable[ZSTD_hashPtr(lowest+1, hBits, mls)] = lowIndex+1;
hashTable[ZSTD_hashPtr(lowest+2, hBits, mls)] = lowIndex+2;
hashTable[ZSTD_hashPtr(lowest+3, hBits, mls)] = lowIndex+3;
}
/* Main Search Loop */
- while (ip < ilimit) /* < instead of <=, because repcode check at (ip+1) */
- {
+ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
size_t mlCode;
size_t offset;
const size_t h = ZSTD_hashPtr(ip, hBits, mls);
const U32 current = (U32)(ip-base);
hashTable[h] = current; /* update hash table */
- if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) /* note : by construction, offset_1 <= current */
- {
+ if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) { /* note : by construction, offset_1 <= current */
mlCode = ZSTD_count(ip+1+MINMATCH, ip+1+MINMATCH-offset_1, iend);
ip++;
offset = 0;
- }
- else
- {
+ } else {
if ( (matchIndex <= lowIndex) ||
- (MEM_read32(match) != MEM_read32(ip)) )
- {
+ (MEM_read32(match) != MEM_read32(ip)) ) {
ip += ((ip-anchor) >> g_searchStrength) + 1;
continue;
}
ip += mlCode + MINMATCH;
anchor = ip;
- if (ip <= ilimit)
- {
+ if (ip <= ilimit) {
/* Fill Table */
hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 */
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
/* check immediate repcode */
while ( (ip <= ilimit)
- && (MEM_read32(ip) == MEM_read32(ip - offset_2)) )
- {
+ && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
/* store sequence */
size_t rlCode = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_2, iend);
size_t tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
ip += rlCode+MINMATCH;
anchor = ip;
continue; /* faster when present ... (?) */
- }
- }
- }
+ } } }
/* Last Literals */
{
/* init */
ZSTD_resetSeqStore(seqStorePtr);
- {
- /* skip first 4 positions to avoid read overflow during repcode match check */
- hashTable[ZSTD_hashPtr(ip+0, hBits, mls)] = (U32)(ip-base+0);
- hashTable[ZSTD_hashPtr(ip+1, hBits, mls)] = (U32)(ip-base+1);
- hashTable[ZSTD_hashPtr(ip+2, hBits, mls)] = (U32)(ip-base+2);
- hashTable[ZSTD_hashPtr(ip+3, hBits, mls)] = (U32)(ip-base+3);
- ip += 4;
- }
+ /* skip first 4 positions to avoid read overflow during repcode match check */
+ hashTable[ZSTD_hashPtr(ip+0, hBits, mls)] = (U32)(ip-base+0);
+ hashTable[ZSTD_hashPtr(ip+1, hBits, mls)] = (U32)(ip-base+1);
+ hashTable[ZSTD_hashPtr(ip+2, hBits, mls)] = (U32)(ip-base+2);
+ hashTable[ZSTD_hashPtr(ip+3, hBits, mls)] = (U32)(ip-base+3);
+ ip += 4;
/* Main Search Loop */
- while (ip < ilimit) /* < instead of <=, because (ip+1) */
- {
+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
const size_t h = ZSTD_hashPtr(ip, hBits, mls);
const U32 matchIndex = hashTable[h];
const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base;
hashTable[h] = current; /* update hash table */
if ( ((repIndex <= dictLimit-4) || (repIndex >= dictLimit))
- && (MEM_read32(repMatch) == MEM_read32(ip+1)) )
- {
+ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
mlCode = ZSTD_count_2segments(ip+1+MINMATCH, repMatch+MINMATCH, iend, repMatchEnd, lowPrefixPtr);
ip++;
offset = 0;
- }
- else
- {
+ } else {
if ( (matchIndex < lowLimit) ||
(MEM_read32(match) != MEM_read32(ip)) )
{ ip += ((ip-anchor) >> g_searchStrength) + 1; continue; }
const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend;
const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
mlCode = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iend, matchEnd, lowPrefixPtr);
- while ((ip>anchor) && (match>lowMatchPtr) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; } /* catch up */
+ while ((ip>anchor) && (match>lowMatchPtr) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; } /* catch up */
offset = current - matchIndex;
offset_2 = offset_1;
offset_1 = offset;
- }
- }
+ } }
/* found a match : store it */
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset, mlCode);
ip += mlCode + MINMATCH;
anchor = ip;
- if (ip <= ilimit)
- {
+ if (ip <= ilimit) {
/* Fill Table */
hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2;
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
/* check immediate repcode */
- while (ip <= ilimit)
- {
+ while (ip <= ilimit) {
U32 current2 = (U32)(ip-base);
const U32 repIndex2 = current2 - offset_2;
const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
if ( ((repIndex2 <= dictLimit-4) || (repIndex2 >= dictLimit))
- && (MEM_read32(repMatch2) == MEM_read32(ip)) )
- {
+ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
size_t repLength2 = ZSTD_count_2segments(ip+MINMATCH, repMatch2+MINMATCH, iend, repEnd2, lowPrefixPtr);
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
continue;
}
break;
- }
- }
- }
+ } } }
/* Last Literals */
{
hashTable[h] = current; /* Update Hash Table */
- while (nbCompares-- && (matchIndex > windowLow))
- {
+ while (nbCompares-- && (matchIndex > windowLow)) {
U32* nextPtr = bt + 2*(matchIndex & btMask);
const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
- if (matchIndex == predictedSmall)
- { /* no need to check length, result known */
+ if (matchIndex == predictedSmall) {
+ /* no need to check length, result known */
*smallerPtr = matchIndex;
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
continue;
}
- if (matchIndex == predictedLarge)
- {
+ if (matchIndex == predictedLarge) {
*largerPtr = matchIndex;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
largerPtr = nextPtr;
continue;
}
- if ((!extDict) || (matchIndex+matchLength >= dictLimit))
- {
+ if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
match = base + matchIndex;
if (match[matchLength] == ip[matchLength])
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1;
- }
- else
- {
+ } else {
match = dictBase + matchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit)
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
- if (match[matchLength] < ip[matchLength]) /* necessarily within correct buffer */
- {
+ if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */
/* match is smaller than current */
*smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
- }
- else
- {
+ } else {
/* match is larger than current */
*largerPtr = matchIndex;
commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
largerPtr = nextPtr;
matchIndex = nextPtr[0];
- }
- }
+ } }
*smallerPtr = *largerPtr = 0;
return (matchEndIdx > current + 8) ? matchEndIdx - current - 8 : 1;
hashTable[h] = current; /* Update Hash Table */
- while (nbCompares-- && (matchIndex > windowLow))
- {
+ while (nbCompares-- && (matchIndex > windowLow)) {
U32* nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match;
- if ((!extDict) || (matchIndex+matchLength >= dictLimit))
- {
+ if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
match = base + matchIndex;
if (match[matchLength] == ip[matchLength])
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1;
- }
- else
- {
+ } else {
match = dictBase + matchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit)
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
}
- if (matchLength > bestLength)
- {
+ if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength;
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) )
break; /* drop, to guarantee consistency (miss a little bit of compression) */
}
- if (match[matchLength] < ip[matchLength])
- {
+ if (match[matchLength] < ip[matchLength]) {
/* match is smaller than current */
*smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
- }
- else
- {
+ } else {
/* match is larger than current */
*largerPtr = matchIndex;
commonLengthLarger = matchLength;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
- while(idx < target)
- {
+ while(idx < target) {
size_t h = ZSTD_hashPtr(base+idx, hashLog, mls);
NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
hashTable[h] = idx;
/* HC4 match finder */
matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);
- while ((matchIndex>lowLimit) && (nbAttempts))
- {
+ while ((matchIndex>lowLimit) && (nbAttempts)) {
size_t currentMl=0;
nbAttempts--;
- if ((!extDict) || matchIndex >= dictLimit)
- {
+ if ((!extDict) || matchIndex >= dictLimit) {
match = base + matchIndex;
if (match[ml] == ip[ml]) /* potentially better */
currentMl = ZSTD_count(ip, match, iLimit);
- }
- else
- {
+ } else {
match = dictBase + matchIndex;
if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH;
if ((ip-base) < REPCODE_STARTVALUE) ip = base + REPCODE_STARTVALUE;
/* Match Loop */
- while (ip < ilimit)
- {
+ while (ip < ilimit) {
size_t matchLength=0;
size_t offset=0;
const BYTE* start=ip+1;
/* check repCode */
- if (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1))
- {
+ if (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1)) {
/* repcode : we take it */
matchLength = ZSTD_count(ip+1+MINMATCH, ip+1+MINMATCH-offset_1, iend) + MINMATCH;
if (depth==0) goto _storeSequence;
matchLength = ml2, start = ip, offset=offsetFound;
}
- if (matchLength < MINMATCH)
- {
+ if (matchLength < MINMATCH) {
ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
continue;
}
/* let's try to find a better solution */
if (depth>=1)
- while (ip<ilimit)
- {
+ while (ip<ilimit) {
ip ++;
- if ((offset) && (MEM_read32(ip) == MEM_read32(ip - offset_1)))
- {
+ if ((offset) && (MEM_read32(ip) == MEM_read32(ip - offset_1))) {
size_t mlRep = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_1, iend) + MINMATCH;
int gain2 = (int)(mlRep * 3);
int gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
size_t ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
- if ((ml2 >= MINMATCH) && (gain2 > gain1))
- {
+ if ((ml2 >= MINMATCH) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */
- }
- }
+ } }
/* let's find an even better one */
- if ((depth==2) && (ip<ilimit))
- {
+ if ((depth==2) && (ip<ilimit)) {
ip ++;
- if ((offset) && (MEM_read32(ip) == MEM_read32(ip - offset_1)))
- {
+ if ((offset) && (MEM_read32(ip) == MEM_read32(ip - offset_1))) {
size_t ml2 = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_1, iend) + MINMATCH;
int gain2 = (int)(ml2 * 4);
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
size_t ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
- if ((ml2 >= MINMATCH) && (gain2 > gain1))
- {
+ if ((ml2 >= MINMATCH) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip;
continue;
- }
- }
- }
+ } } }
break; /* nothing found : store previous solution */
}
/* catch up */
- if (offset)
- {
+ if (offset) {
while ((start>anchor) && (start>base+offset) && (start[-1] == start[-1-offset])) /* only search for offset within prefix */
{ start--; matchLength++; }
offset_2 = offset_1; offset_1 = offset;
/* check immediate repcode */
while ( (ip <= ilimit)
- && (MEM_read32(ip) == MEM_read32(ip - offset_2)) )
- {
+ && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
/* store sequence */
matchLength = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_2, iend);
offset = offset_2;
ip += matchLength+MINMATCH;
anchor = ip;
continue; /* faster when present ... (?) */
- }
- }
+ } }
/* Last Literals */
{
if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
/* Match Loop */
- while (ip < ilimit)
- {
+ while (ip < ilimit) {
size_t matchLength=0;
size_t offset=0;
const BYTE* start=ip+1;
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
- if (MEM_read32(ip+1) == MEM_read32(repMatch))
- {
+ if (MEM_read32(ip+1) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+1+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH;
if (depth==0) goto _storeSequence;
- }
- }
+ } }
{
/* first search (depth 0) */
matchLength = ml2, start = ip, offset=offsetFound;
}
- if (matchLength < MINMATCH)
- {
+ if (matchLength < MINMATCH) {
ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
continue;
}
/* let's try to find a better solution */
if (depth>=1)
- while (ip<ilimit)
- {
+ while (ip<ilimit) {
ip ++;
current++;
/* check repCode */
- if (offset)
- {
+ if (offset) {
const U32 repIndex = (U32)(current - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
- if (MEM_read32(ip) == MEM_read32(repMatch))
- {
+ if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t repLength = ZSTD_count_2segments(ip+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH;
int gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
if ((repLength >= MINMATCH) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip;
- }
- }
+ } }
/* search match, depth 1 */
{
size_t ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
- if ((ml2 >= MINMATCH) && (gain2 > gain1))
- {
+ if ((ml2 >= MINMATCH) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */
- }
- }
+ } }
/* let's find an even better one */
- if ((depth==2) && (ip<ilimit))
- {
+ if ((depth==2) && (ip<ilimit)) {
ip ++;
current++;
/* check repCode */
- if (offset)
- {
+ if (offset) {
const U32 repIndex = (U32)(current - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
- if (MEM_read32(ip) == MEM_read32(repMatch))
- {
+ if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t repLength = ZSTD_count_2segments(ip+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH;
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
if ((repLength >= MINMATCH) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip;
- }
- }
+ } }
/* search match, depth 2 */
{
size_t ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
- if ((ml2 >= MINMATCH) && (gain2 > gain1))
- {
+ if ((ml2 >= MINMATCH) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip;
continue;
- }
- }
- }
+ } } }
break; /* nothing found : store previous solution */
}
/* catch up */
- if (offset)
- {
+ if (offset) {
U32 matchIndex = (U32)((start-base) - offset);
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
}
/* check immediate repcode */
- while (ip <= ilimit)
- {
+ while (ip <= ilimit) {
const U32 repIndex = (U32)((ip-base) - offset_2);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
- if (MEM_read32(ip) == MEM_read32(repMatch))
- {
+ if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH;
continue; /* faster when present ... (?) */
}
break;
- }
- }
+ } }
/* Last Literals */
{
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
/* note : magic number already checked */
+ size_t offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize, errorCode;
+ short offcodeNCount[MaxOff+1];
+ unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
+ short matchlengthNCount[MaxML+1];
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ short litlengthNCount[MaxLL+1];
+ unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+
const size_t hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
- zc->flagHufTable = 1;
- return hufHeaderSize;
+ zc->flagStaticTables = 1;
+ dict = (const char*)dict + hufHeaderSize;
+ dictSize -= hufHeaderSize;
+
+ offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
+ if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
+ if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
+ dict = (const char*)dict + offcodeHeaderSize;
+ dictSize -= offcodeHeaderSize;
+
+ matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
+ if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
+ if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
+ dict = (const char*)dict + matchlengthHeaderSize;
+ dictSize -= matchlengthHeaderSize;
+
+ litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
+ if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
+ if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
+
+ return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
}
+
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
if (dict && dictSize)
size_t hbSize = 0;
/* empty frame */
- if (zc->stage==0)
- {
+ if (zc->stage==0) {
hbSize = zc->hbSize;
if (maxDstSize <= hbSize) return ERROR(dstSize_tooSmall);
zc->stage = 1;
#endif
-/* *******************************************************
+/*-*******************************************************
* Includes
*********************************************************/
#include <stdlib.h> /* calloc */
# include "zstd_legacy.h"
#endif
-/* *******************************************************
+
+/*-*******************************************************
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
#endif
-/* *************************************
+/*-*************************************
* Local types
***************************************/
typedef struct
struct ZSTD_DCtx_s
{
- U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
- U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
- U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
- U32 hufTableX4[HUF_DTABLE_SIZE(HufLog)];
+ FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
+ FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+ FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+ unsigned hufTableX4[HUF_DTABLE_SIZE(HufLog)];
const void* previousDstEnd;
const void* base;
const void* vBase;
size_t expected;
size_t headerSize;
ZSTD_parameters params;
- blockType_t bType;
+ blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage;
- U32 flagHufTable;
+ U32 flagStaticTables;
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
+size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); }
+
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
{
dctx->expected = ZSTD_frameHeaderSize_min;
dctx->vBase = NULL;
dctx->dictEnd = NULL;
dctx->hufTableX4[0] = HufLog;
- dctx->flagHufTable = 0;
+ dctx->flagStaticTables = 0;
return 0;
}
*/
/* Block format description
- Literal Section - Sequences Section
+
+ Block = Literal Section - Sequences Section
+ Prerequisite : size of (compressed) block, maximum size of regenerated data
+
1) Literal Section
- 1.1) Header : up to 5 bytes
- flags:
+
+ 1.1) Header : 1-5 bytes
+ flags: 2 bits
00 compressed by Huff0
- 01 is Raw (uncompressed)
- 10 is Rle
- 11 unused
- Note : using 11 for Huff0 with precomputed table ?
+ 01 unused
+ 10 is Raw (uncompressed)
+ 11 is Rle
+ Note : using 01 => Huff0 with precomputed table ?
Note : delta map ? => compressed ?
- Note 2 : 19 bits for sizes, seems a bit larger than necessary
- Note 3 : RLE blocks ?
+
+ 1.1.1) Huff0-compressed literal block : 3-5 bytes
+ srcSize < 1 KB => 3 bytes (2-2-10-10) => single stream
+ srcSize < 1 KB => 3 bytes (2-2-10-10)
+ srcSize < 16KB => 4 bytes (2-2-14-14)
+ else => 5 bytes (2-2-18-18)
+ big endian convention
+
+ 1.1.2) Raw (uncompressed) literal block header : 1-3 bytes
+ size : 5 bits: (IS_RAW<<6) + (0<<4) + size
+ 12 bits: (IS_RAW<<6) + (2<<4) + (size>>8)
+ size&255
+ 20 bits: (IS_RAW<<6) + (3<<4) + (size>>16)
+ size>>8&255
+ size&255
+
+ 1.1.3) Rle (repeated single byte) literal block header : 1-3 bytes
+ size : 5 bits: (IS_RLE<<6) + (0<<4) + size
+ 12 bits: (IS_RLE<<6) + (2<<4) + (size>>8)
+ size&255
+ 20 bits: (IS_RLE<<6) + (3<<4) + (size>>16)
+ size>>8&255
+ size&255
+
+ 1.1.4) Huff0-compressed literal block, using precomputed CTables : 3-5 bytes
+ srcSize < 1 KB => 3 bytes (2-2-10-10) => single stream
+ srcSize < 1 KB => 3 bytes (2-2-10-10)
+ srcSize < 16KB => 4 bytes (2-2-14-14)
+ else => 5 bytes (2-2-18-18)
+ big endian convention
+
+ 1- CTable available (stored into workspace ?)
+ 2- Small input (fast heuristic ? Full comparison ? depend on clevel ?)
+
+
+ 1.2) Literal block content
1.2.1) Huff0 block, using sizes from header
See Huff0 format
- 1.2.2) Huff0 block, using precomputed DTable
- _usingDTable variants
+ 1.2.2) Huff0 block, using prepared table
+
+ 1.2.3) Raw content
+
+ 1.2.4) single byte
- 1.2.3) uncompressed blocks
- as the name says (both 2 or 3 bytes variants)
2) Sequences section
TO DO
litCSize = ((istart[2] & 3) << 16) + (istart[3] << 8) + istart[4];
break;
}
- if (litSize > BLOCKSIZE) return ERROR(corruption_detected);;
+ if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
if (HUF_isError(singleStream ?
HUF_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) :
U32 lhSize = ((istart[0]) >> 4) & 3;
if (lhSize != 1) /* only case supported for now : small litSize, single stream */
return ERROR(corruption_detected);
- if (!dctx->flagHufTable)
+ if (!dctx->flagStaticTables)
return ERROR(dictionary_corrupted);
/* 2 - 2 - 10 - 10 */
LLtype = *ip >> 6;
Offtype = (*ip >> 4) & 3;
MLtype = (*ip >> 2) & 3;
- if (*ip & 2)
- {
+ if (*ip & 2) {
dumpsLength = ip[2];
dumpsLength += ip[1] << 8;
ip += 3;
- }
- else
- {
+ } else {
dumpsLength = ip[1];
dumpsLength += (ip[0] & 1) << 8;
ip += 2;
switch(LLtype)
{
U32 max;
- case bt_rle :
+ case FSE_ENCODING_RLE :
LLlog = 0;
- FSE_buildDTable_rle(DTableLL, *ip++); break;
- case bt_raw :
+ FSE_buildDTable_rle(DTableLL, *ip++);
+ break;
+ case FSE_ENCODING_RAW :
LLlog = LLbits;
- FSE_buildDTable_raw(DTableLL, LLbits); break;
- default :
+ FSE_buildDTable_raw(DTableLL, LLbits);
+ break;
+ case FSE_ENCODING_STATIC:
+ break;
+ case FSE_ENCODING_DYNAMIC :
+ default : /* impossible */
max = MaxLL;
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return ERROR(GENERIC);
switch(Offtype)
{
U32 max;
- case bt_rle :
+ case FSE_ENCODING_RLE :
Offlog = 0;
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
FSE_buildDTable_rle(DTableOffb, *ip++ & MaxOff); /* if *ip > MaxOff, data is corrupted */
break;
- case bt_raw :
+ case FSE_ENCODING_RAW :
Offlog = Offbits;
- FSE_buildDTable_raw(DTableOffb, Offbits); break;
- default :
+ FSE_buildDTable_raw(DTableOffb, Offbits);
+ break;
+ case FSE_ENCODING_STATIC:
+ break;
+ case FSE_ENCODING_DYNAMIC :
+ default : /* impossible */
max = MaxOff;
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
if (FSE_isError(headerSize)) return ERROR(GENERIC);
switch(MLtype)
{
U32 max;
- case bt_rle :
+ case FSE_ENCODING_RLE :
MLlog = 0;
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
- FSE_buildDTable_rle(DTableML, *ip++); break;
- case bt_raw :
+ FSE_buildDTable_rle(DTableML, *ip++);
+ break;
+ case FSE_ENCODING_RAW :
MLlog = MLbits;
- FSE_buildDTable_raw(DTableML, MLbits); break;
- default :
+ FSE_buildDTable_raw(DTableML, MLbits);
+ break;
+ case FSE_ENCODING_STATIC:
+ break;
+ case FSE_ENCODING_DYNAMIC :
+ default : /* impossible */
max = MaxML;
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return ERROR(GENERIC);
if (MLlog > MLFSELog) return ERROR(corruption_detected);
ip += headerSize;
FSE_buildDTable(DTableML, norm, max, MLlog);
- }
- }
+ } }
return ip-istart;
}
/* Literal length */
litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
prevOffset = litLength ? seq->offset : seqState->prevOffset;
- if (litLength == MaxLL)
- {
+ if (litLength == MaxLL) {
U32 add = *dumps++;
if (add < 255) litLength += add;
- else
- {
+ else {
litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
dumps += 3;
}
/* MatchLength */
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
- if (matchLength == MaxML)
- {
+ if (matchLength == MaxML) {
U32 add = *dumps++;
if (add < 255) matchLength += add;
- else
- {
+ else {
matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
dumps += 3;
}
seq->offset = offset;
seq->matchLength = matchLength;
seqState->dumps = dumps;
+
+#if 0
+ {
+ static U64 totalDecoded = 0;
+ printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n",
+ (U32)(totalDecoded), (U32)litLength, (U32)matchLength, (U32)offset);
+ totalDecoded += litLength + matchLength;
+ }
+#endif
}
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
- }
- }
+ } }
/* match within prefix */
if (sequence.offset < 8) {
match += dec32table[sequence.offset];
ZSTD_copy4(op+4, match);
match -= sub2;
- }
- else {
+ } else {
ZSTD_copy8(op, match);
}
op += 8; match += 8;
match += oend_8 - op;
op = oend_8;
}
- while (op < oMatchEnd) *op++ = *match++;
- }
- else {
+ while (op < oMatchEnd)
+ *op++ = *match++;
+ } else {
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
/* Build Decoding Tables */
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
DTableLL, DTableML, DTableOffb,
- ip, iend-ip);
+ ip, seqSize);
if (ZSTD_isError(errorCode)) return errorCode;
ip += errorCode;
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
if (op != litPtr) memcpy(op, litPtr, lastLLSize);
op += lastLLSize;
- }
- }
+ } }
return op-ostart;
}
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
- size_t hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
+ size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
+ short offcodeNCount[MaxOff+1];
+ U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
+ short matchlengthNCount[MaxML+1];
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ short litlengthNCount[MaxLL+1];
+ unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+
+ hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
- dctx->flagHufTable = 1;
- return hSize;
+ dict = (const char*)dict + hSize;
+ dictSize -= hSize;
+
+ offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
+ if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
+ if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
+ dict = (const char*)dict + offcodeHeaderSize;
+ dictSize -= offcodeHeaderSize;
+
+ matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
+ if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
+ if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
+ dict = (const char*)dict + matchlengthHeaderSize;
+ dictSize -= matchlengthHeaderSize;
+
+ litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
+ if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
+ if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
+
+ dctx->flagStaticTables = 1;
+ return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
}
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
projects / thirdparty / zstd.git / commitdiff
