case ZSTD_error_tableLog_tooLarge: return "tableLog requires too much memory";
case ZSTD_error_maxSymbolValue_tooLarge: return "Unsupported max possible Symbol Value : too large";
case ZSTD_error_maxSymbolValue_tooSmall: return "Specified maxSymbolValue is too small";
+ case ZSTD_error_dictionary_corrupted: return "Dictionary is corrupted";
case ZSTD_error_maxCode:
default: return codeError;
}
ZSTD_error_tableLog_tooLarge,
ZSTD_error_maxSymbolValue_tooLarge,
ZSTD_error_maxSymbolValue_tooSmall,
+ ZSTD_error_dictionary_corrupted,
ZSTD_error_maxCode
};
@dst : destination buffer
@CTable : huffman tree to save, using huff0 representation
@return : size of saved CTable */
-size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
+size_t HUF_writeCTable (void* dst, size_t maxDstSize,
+ const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
{
BYTE bitsToWeight[HUF_MAX_TABLELOG + 1];
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
}
+static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize);
+
+
+size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
+{
+ BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
+ U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
+ U32 tableLog = 0;
+ size_t iSize;
+ U32 nbSymbols = 0;
+ U32 n;
+ U32 nextRankStart;
+ //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
+
+ /* get symbol weights */
+ iSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
+ if (HUF_isError(iSize)) return iSize;
+
+ /* check result */
+ if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
+ if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
+
+ /* Prepare base value per rank */
+ nextRankStart = 0;
+ for (n=1; n<=tableLog; n++) {
+ U32 current = nextRankStart;
+ nextRankStart += (rankVal[n] << (n-1));
+ rankVal[n] = current;
+ }
+
+ /* fill nbBits */
+ for (n=0; n<nbSymbols; n++) {
+ const U32 w = huffWeight[n];
+ CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
+ }
+
+ /* fill val */
+ {
+ U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
+ U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
+ for (n=0; n<nbSymbols; n++)
+ nbPerRank[CTable[n].nbBits]++;
+ {
+ /* determine stating value per rank */
+ U16 min = 0;
+ for (n=HUF_MAX_TABLELOG; n>0; n--)
+ {
+ valPerRank[n] = min; /* get starting value within each rank */
+ min += nbPerRank[n];
+ min >>= 1;
+ }
+ }
+ for (n=0; n<=maxSymbolValue; n++)
+ CTable[n].val = valPerRank[CTable[n].nbBits]++; /* assign value within rank, symbol order */
+ }
+
+ return iSize;
+}
+
+
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
{
int totalCost = 0;
#define HUF_FLUSHBITS_2(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream)
-size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
const BYTE* ip = (const BYTE*) src;
BYTE* const ostart = (BYTE*)dst;
}
-static size_t HUF_compress_into4Segments(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
size_t segmentSize = (srcSize+3)/4; /* first 3 segments */
size_t errorCode;
if (srcSize < 12) return 0; /* no saving possible : too small input */
op += 6; /* jumpTable */
- errorCode = HUF_compress_usingCTable(op, oend-op, ip, segmentSize, CTable);
+ errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
MEM_writeLE16(ostart, (U16)errorCode);
ip += segmentSize;
op += errorCode;
- errorCode = HUF_compress_usingCTable(op, oend-op, ip, segmentSize, CTable);
+ errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
MEM_writeLE16(ostart+2, (U16)errorCode);
ip += segmentSize;
op += errorCode;
- errorCode = HUF_compress_usingCTable(op, oend-op, ip, segmentSize, CTable);
+ errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
MEM_writeLE16(ostart+4, (U16)errorCode);
ip += segmentSize;
op += errorCode;
- errorCode = HUF_compress_usingCTable(op, oend-op, ip, iend-ip, CTable);
+ errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
size_t errorCode;
/* checks & inits */
- if (srcSize < 2) return 0; /* Uncompressed */
+ if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */
if (dstSize < 1) return 0; /* not compressible within dst budget */
if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */
if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Compress */
if (singleStream)
- errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */
+ errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */
else
- errorCode = HUF_compress_into4Segments(op, oend - op, src, srcSize, CTable);
+ errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
op += errorCode;
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog)
{
-#if 1
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
-#else
- BYTE* const ostart = (BYTE*)dst;
- BYTE* op = ostart;
- BYTE* const oend = ostart + dstSize;
-
- U32 count[HUF_MAX_SYMBOL_VALUE+1];
- HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1];
- size_t errorCode;
-
- /* checks & inits */
- if (srcSize < 1) return 0; /* Uncompressed */
- if (dstSize < 1) return 0; /* not compressible within dst budget */
- if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */
- if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
- if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
- if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
-
- /* Scan input and build symbol stats */
- errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
- if (HUF_isError(errorCode)) return errorCode;
- if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }
- if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */
-
- /* Build Huffman Tree */
- errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
- if (HUF_isError(errorCode)) return errorCode;
- huffLog = (U32)errorCode;
-
- /* Write table description header */
- errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
- if (HUF_isError(errorCode)) return errorCode;
- if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */
- op += errorCode;
-
- /* Compress */
- //if (srcSize < MIN_4STREAMS) errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable); else /* single segment */
- errorCode = HUF_compress_into4Segments(op, oend - op, src, srcSize, CTable);
- if (HUF_isError(errorCode)) return errorCode;
- if (errorCode==0) return 0;
- op += errorCode;
-
- /* check compressibility */
- if ((size_t)(op-ostart) >= srcSize-1)
- return 0;
-
- return op-ostart;
-#endif
}
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
- if (iSize >= 128) /* special header */
- {
- if (iSize >= (242)) /* RLE */
- {
+ if (iSize >= 128) { /* special header */
+ if (iSize >= (242)) { /* RLE */
static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
oSize = l[iSize-242];
memset(huffWeight, 1, hwSize);
iSize = 0;
}
- else /* Incompressible */
- {
+ else { /* Incompressible */
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
- for (n=0; n<oSize; n+=2)
- {
+ for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
- }
- }
- }
- else /* header compressed with FSE (normal case) */
- {
+ } } }
+ else { /* header compressed with FSE (normal case) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize)) return oSize;
/* check result */
if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */
- DTable[0] = (U16)tableLog; /* maybe should separate sizeof DTable, as allocated, from used size of DTable, in case of DTable re-use */
+ DTable[0] = (U16)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */
/* Prepare ranks */
nextRankStart = 0;
For example, it's possible to compress several blocks using the same 'CTable',
or to save and regenerate 'CTable' using external methods.
*/
-
/* FSE_count() : find it within "fse.h" */
-
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
-size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
-size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, unsigned maxSymbolValue, unsigned huffLog);
-size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
+size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
+size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
+size_t HUF_compress4X_into4Segments(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/*!
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
-
*/
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
/* single stream variants */
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
+/* Loading a CTable saved with HUF_writeCTable() */
+
+size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
+
+
#if defined (__cplusplus)
}
#endif
seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable;
U32* contentTable;
+ HUF_CElt* hufTable;
+ U32 flagHufTable;
};
{
const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog)) * sizeof(U32);
- const size_t neededSpace = tableSpace + (3*blockSize);
+ const size_t neededSpace = tableSpace + (256*sizeof(U32)) + (3*blockSize);
if (zc->workSpaceSize < neededSpace)
{
free(zc->workSpace);
- zc->workSpaceSize = neededSpace;
zc->workSpace = malloc(neededSpace);
if (zc->workSpace == NULL) return ERROR(memory_allocation);
+ zc->workSpaceSize = neededSpace;
}
- memset(zc->workSpace, 0, tableSpace );
+ memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
zc->hashTable = (U32*)(zc->workSpace);
zc->contentTable = zc->hashTable + ((size_t)1 << params.hashLog);
- zc->seqStore.buffer = (void*) (zc->contentTable + ((size_t)1 << contentLog));
+ zc->seqStore.buffer = zc->contentTable + ((size_t)1 << contentLog);
+ zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
+ zc->flagHufTable = 0;
+ zc->seqStore.buffer = (U32*)(zc->seqStore.buffer) + 256;
}
zc->nextToUpdate = 1;
big endian convention
1- CTable available (stored into workspace ?)
- 2- Small input
+ 2- Small input (fast heuristic ? Full comparison ? depend on clevel ?)
1.2) Literal block content
}
-size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 1; }
+size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
-static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
+static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
+ void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
const size_t minGain = ZSTD_minGain(srcSize);
BYTE* const ostart = (BYTE*)dst;
- size_t lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
+ const size_t lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
U32 singleStream = srcSize < 256;
+ U32 hType = IS_HUF;
size_t clitSize;
if (maxDstSize < 4) return ERROR(dstSize_tooSmall); /* not enough space for compression */
- clitSize = singleStream ? HUF_compress1X(ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12)
- : HUF_compress2 (ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12);
+ if (zc->flagHufTable && (lhSize==3))
+ {
+ hType = IS_PCH;
+ singleStream = 1;
+ clitSize = HUF_compress1X_usingCTable(ostart+lhSize, maxDstSize-lhSize, src, srcSize, zc->hufTable);
+ }
+ else
+ {
+ clitSize = singleStream ? HUF_compress1X(ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12)
+ : HUF_compress2 (ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12);
+ }
if ((clitSize==0) || (clitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, maxDstSize, src, srcSize);
if (clitSize==1) return ZSTD_compressRleLiteralsBlock(dst, maxDstSize, src, srcSize);
switch(lhSize)
{
case 3: /* 2 - 2 - 10 - 10 */
- ostart[0] = (BYTE)((srcSize>>6) + (singleStream << 4));
+ ostart[0] = (BYTE)((srcSize>>6) + (singleStream << 4) + (hType<<6));
ostart[1] = (BYTE)((srcSize<<2) + (clitSize>>8));
ostart[2] = (BYTE)(clitSize);
break;
case 4: /* 2 - 2 - 14 - 14 */
- ostart[0] = (BYTE)(srcSize>>10) + (2<<4);
+ ostart[0] = (BYTE)((srcSize>>10) + (2<<4) + (hType<<6));
ostart[1] = (BYTE)(srcSize>> 2);
ostart[2] = (BYTE)((srcSize<<6) + (clitSize>>8));
ostart[3] = (BYTE)(clitSize);
break;
default: /* should not be necessary, lhSize is {3,4,5} */
case 5: /* 2 - 2 - 18 - 18 */
- ostart[0] = (BYTE)(srcSize>>14) + (3<<4);
+ ostart[0] = (BYTE)((srcSize>>14) + (3<<4) + (hType<<6));
ostart[1] = (BYTE)(srcSize>>6);
ostart[2] = (BYTE)((srcSize<<2) + (clitSize>>16));
ostart[3] = (BYTE)(clitSize>>8);
#define LITERAL_NOENTROPY 63 /* don't even attempt to compress literals below this threshold (cheap heuristic) */
-size_t ZSTD_compressSequences(void* dst, size_t maxDstSize,
- const seqStore_t* seqStorePtr,
+size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
+ void* dst, size_t maxDstSize,
size_t srcSize)
{
+ const seqStore_t* seqStorePtr = &(zc->seqStore);
U32 count[MaxSeq+1];
S16 norm[MaxSeq+1];
size_t mostFrequent;
{
size_t cSize;
size_t litSize = seqStorePtr->lit - op_lit_start;
+ const size_t minLitSize = zc->flagHufTable ? 6 : LITERAL_NOENTROPY;
- if (litSize <= LITERAL_NOENTROPY)
+ if (litSize <= minLitSize)
cSize = ZSTD_noCompressLiterals(op, maxDstSize, op_lit_start, litSize);
else
- cSize = ZSTD_compressLiterals(op, maxDstSize, op_lit_start, litSize);
+ cSize = ZSTD_compressLiterals(zc, op, maxDstSize, op_lit_start, litSize);
if (ZSTD_isError(cSize)) return cSize;
op += cSize;
}
typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize);
-ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
+static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
{
static const ZSTD_blockCompressor blockCompressor[2][5] = {
{ ZSTD_compressBlock_fast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy,ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2 },
ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit);
if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */
blockCompressor(zc, src, srcSize);
- return ZSTD_compressSequences(dst, maxDstSize, &(zc->seqStore), srcSize);
+ return ZSTD_compressSequences(zc, dst, maxDstSize, srcSize);
}
}
-size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize)
+static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize)
{
const BYTE* const ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
}
+/* Dictionary format :
+ Magic == ZSTD_DICT_MAGIC (4 bytes)
+ Huff0 CTable (256 * 4 bytes) => to be changed to read from writeCTable
+ Dictionary content
+*/
+/*! ZSTD_loadDictEntropyStats
+ @return : size read from dictionary */
+static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
+{
+ /* note : magic number already checked */
+ const size_t hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
+ if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
+ zc->flagHufTable = 1;
+ return hufHeaderSize;
+}
+
+size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
+{
+ U32 magic = MEM_readLE32(dict);
+ U32 eSize;
+ if (magic != ZSTD_DICT_MAGIC)
+ return ZSTD_loadDictionaryContent(zc, dict, dictSize);
+
+ eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4, dictSize-4) + 4;
+ if (ZSTD_isError(eSize)) return eSize;
+ return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
+}
+
+
/*! ZSTD_duplicateCCtx
* Duplicate an existing context @srcCCtx into another one @dstCCtx.
* Only works during stage 0 (i.e. before first call to ZSTD_compressContinue())
dstCCtx->dictLimit = srcCCtx->dictLimit;
dstCCtx->lowLimit = srcCCtx->lowLimit;
+ dstCCtx->flagHufTable = srcCCtx->flagHufTable;
+ if (dstCCtx->flagHufTable)
+ memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
+
return 0;
}
return result;
}
+/* to do
+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, int compressionLevel)
+{
+ return 0;
+}*/
size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, int compressionLevel)
{
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+ U32 hufTableX4[HUF_DTABLE_SIZE(HufLog)];
const void* previousDstEnd;
const void* base;
const void* vBase;
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
- BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
+ BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
dctx->base = NULL;
dctx->vBase = NULL;
dctx->dictEnd = NULL;
+ dctx->hufTableX4[0] = HufLog;
return 0;
}
dctx->litSize = litSize;
return litCSize + lhSize;
}
+ case IS_PCH:
+ {
+ size_t errorCode;
+ size_t litSize, litCSize;
+ U32 lhSize = ((istart[0]) >> 4) & 3;
+ if (lhSize != 1) /* only case supported for now : small litSize, single stream */
+ return ERROR(corruption_detected);
+
+ /* 2 - 2 - 10 - 10 */
+ lhSize=3;
+ litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
+ litCSize = ((istart[1] & 3) << 8) + istart[2];
+
+ errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
+ if (HUF_isError(errorCode)) return ERROR(corruption_detected);
+
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
+ dctx->litSize = litSize;
+ return litCSize + lhSize;
+ }
case IS_RAW:
{
size_t litSize;
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+8;
+ dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
- default: /* IS_PCH */
- return ERROR(corruption_detected); /* not yet nominal case */
+ default:
+ return ERROR(corruption_detected); /* impossible */
}
}
ZSTD_resetDCtx(dctx);
if (dict)
{
- ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
+ size_t errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
+ if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted);
dctx->dictEnd = dctx->previousDstEnd;
dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
dctx->base = dst;
}
-void ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
dctx->dictEnd = dctx->previousDstEnd;
dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
dctx->base = dict;
dctx->previousDstEnd = (const char*)dict + dictSize;
}
+
+
+static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ size_t hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
+ if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
+ return hSize;
+}
+
+size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ size_t eSize;
+ U32 magic = MEM_readLE32(dict);
+ if (magic != ZSTD_DICT_MAGIC) {
+ /* pure content mode */
+ ZSTD_refDictContent(dctx, dict, dictSize);
+ return 0;
+ }
+ /* load entropy tables */
+ dict = (const char*)dict + 4;
+ dictSize -= 4;
+ eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
+ if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
+
+ /* reference dictionary content */
+ dict = (const char*)dict + eSize;
+ dictSize -= eSize;
+ ZSTD_refDictContent(dctx, dict, dictSize);
+
+ return 0;
+}
+
/* *************************************
* Common constants
***************************************/
-#define ZSTD_MAGICNUMBER 0xFD2FB524 /* v0.4 */
+#define ZSTD_MAGICNUMBER 0xFD2FB525 /* v0.5 */
+#define ZSTD_DICT_MAGIC 0xEC30A435
#define KB *(1 <<10)
#define MB *(1 <<20)
#define OffFSELog 9
#define MaxSeq MAX(MaxLL, MaxML)
+#define HufLog 12
+
#define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + MIN_SEQUENCES_SIZE)
* Streaming functions (direct mode)
****************************************/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, int compressionLevel);
+//ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params);
+
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters params);
ZSTDLIB_API size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
-ZSTDLIB_API void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
+ZSTDLIB_API size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
if (ZSTD_isError(rSize)) EXM_THROW(2, "ZSTD_compressEnd() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].cSize += rSize;
}
- /*blockTable[blockNb].cSize = ZSTD_compress_usingDict(ctx,
- blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
- blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
- dictBuffer, dictBufferSize,
- cLevel);*/
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
- for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++)
- {
- for (blockNb=0; blockNb<nbBlocks; blockNb++)
+ for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++) {
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
blockTable[blockNb].resSize = ZSTD_decompress_usingDict(dctx,
- blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
- blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
- dictBuffer, dictBufferSize);
- }
+ blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
+ dictBuffer, dictBufferSize);
+ if (ZSTD_isError(blockTable[blockNb].resSize))
+ EXM_THROW(3, "ZSTD_decompress_usingDict() failed : %s", ZSTD_getErrorName(blockTable[blockNb].resSize));
+ } }
milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
projects / thirdparty / zstd.git / commitdiff
