for (i=0; i<tableSize; i++)
{
FSE_FUNCTION_TYPE s = tableSymbol[i]; /* static analyzer doesn't understand tableSymbol is properly initialized */
- tableU16[cumul[s]++] = (U16) (tableSize+i); /* Table U16 : sorted by symbol order; gives next state value */
+ tableU16[cumul[s]++] = (U16) (tableSize+i); /* TableU16 : sorted by symbol order; gives next state value */
}
/* Build Symbol Transformation Table */
break;
case -1:
case 1:
- symbolTT[s].deltaNbBits = tableLog * (1 << 16);
+ symbolTT[s].deltaNbBits = tableLog << 16;
symbolTT[s].deltaFindState = total - 1;
total ++;
break;
free(dt);
}
+typedef struct {
+ U16 tableLog;
+ U16 fastMode;
+} FSE_DTableHeader; /* sizeof U32 */
size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
- U32* const base32 = (U32*)dt;
- FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (base32+1);
+ FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
+ FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */
const U32 tableSize = 1 << tableLog;
const U32 tableMask = tableSize-1;
const U32 step = FSE_tableStep(tableSize);
if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_tableLog_tooLarge;
/* Init, lay down lowprob symbols */
- base32[0] = tableLog;
+ DTableH[0].tableLog = (U16)tableLog;
for (s=0; s<=maxSymbolValue; s++)
{
if (normalizedCounter[s]==-1)
}
}
- return noLarge;
+ DTableH->fastMode = (U16)noLarge;
+ return 0;
}
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
{
size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 1;
- return maxSymbolValue ? maxHeaderSize : FSE_MAX_HEADERSIZE;
+ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND;
}
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
{
start+=24;
bitStream += 0xFFFFU << bitCount;
- if ((!safeWrite) && (out > oend-2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
+ if ((!safeWrite) && (out > oend-2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
out[0] = (BYTE) bitStream;
out[1] = (BYTE)(bitStream>>8);
out+=2;
bitCount += 2;
if (bitCount>16)
{
- if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
+ if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out += 2;
}
if (bitCount>16)
{
- if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
+ if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out += 2;
}
/* flush remaining bitStream */
- if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
+ if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out+= (bitCount+7) /8;
}
-typedef struct
-{
- U32 id;
- U32 count;
-} rank_t;
-
-int FSE_compareRankT(const void* r1, const void* r2)
-{
- const rank_t* R1 = (const rank_t*)r1;
- const rank_t* R2 = (const rank_t*)r2;
-
- return 2 * (R1->count < R2->count) - 1;
-}
-
-
/* Secondary normalization method.
To be used when primary method fails. */
}
-void FSE_initCStream(FSE_CStream_t* bitC, void* start)
+size_t FSE_initCStream(FSE_CStream_t* bitC, void* start, size_t maxSize)
{
+ if (maxSize < 8) return (size_t)-FSE_ERROR_dstSize_tooSmall;
bitC->bitContainer = 0;
- bitC->bitPos = 0; /* reserved for unusedBits */
+ bitC->bitPos = 0;
bitC->startPtr = (char*)start;
bitC->ptr = bitC->startPtr;
+ bitC->endPtr = bitC->startPtr + maxSize - sizeof(bitC->ptr);
+ return 0;
}
void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
-void FSE_flushBits(FSE_CStream_t* bitC)
+void FSE_flushBitsFast(FSE_CStream_t* bitC) /* only if dst buffer is large enough ( >= FSE_compressBound()) */
{
size_t nbBytes = bitC->bitPos >> 3;
FSE_writeLEST(bitC->ptr, bitC->bitContainer);
- bitC->bitPos &= 7;
bitC->ptr += nbBytes;
+ bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes*8;
}
+void FSE_flushBits(FSE_CStream_t* bitC)
+{
+ size_t nbBytes = bitC->bitPos >> 3;
+ FSE_writeLEST(bitC->ptr, bitC->bitContainer);
+ bitC->ptr += nbBytes;
+ if (bitC->ptr <= bitC->endPtr)
+ {
+ bitC->bitPos &= 7;
+ bitC->bitContainer >>= nbBytes*8;
+ return;
+ }
+}
+
void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* statePtr)
{
FSE_addBits(bitC, statePtr->value, statePtr->stateLog);
{
char* endPtr;
- FSE_addBits(bitC, 1, 1);
+ FSE_addBitsFast(bitC, 1, 1);
FSE_flushBits(bitC);
+ if (bitC->bitPos > 7) /* still some data to flush => too close to buffer's end */
+ return 0; /* not compressible */
+
endPtr = bitC->ptr;
endPtr += bitC->bitPos > 0;
}
-size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
+static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
const void* src, size_t srcSize,
- const FSE_CTable* ct)
+ const FSE_CTable* ct, const unsigned fast)
{
const BYTE* const istart = (const BYTE*) src;
const BYTE* ip;
const BYTE* const iend = istart + srcSize;
+ size_t errorCode;
FSE_CStream_t bitC;
FSE_CState_t CState1, CState2;
/* init */
- (void)dstSize; /* objective : ensure it fits into dstBuffer (Todo) */
- FSE_initCStream(&bitC, dst);
+ errorCode = FSE_initCStream(&bitC, dst, dstSize);
+ if (FSE_isError(errorCode)) return 0;
FSE_initCState(&CState1, ct);
CState2 = CState1;
ip=iend;
+#define FSE_FLUSHBITS(s) (fast ? FSE_flushBitsFast(s) : FSE_flushBits(s))
+
/* join to even */
if (srcSize & 1)
{
FSE_encodeSymbol(&bitC, &CState1, *--ip);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
}
/* join to mod 4 */
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
}
/* 2 or 4 encoding per loop */
- while (ip>istart)
+ for ( ; ip>istart ; )
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
}
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
}
FSE_flushCState(&bitC, &CState2);
return FSE_closeCStream(&bitC);
}
+size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const FSE_CTable* ct)
+{
+ const unsigned fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
-size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
+ if (fast)
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
+ else
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
+}
+size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
+
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
{
const BYTE* const istart = (const BYTE*) src;
CTable_max_t ct;
size_t errorCode;
- /* early out */
- if (dstSize < FSE_compressBound(srcSize)) return (size_t)-FSE_ERROR_dstSize_tooSmall;
- if (srcSize <= 1) return srcSize; /* Uncompressed or RLE */
+ /* init conditions */
+ if (srcSize <= 1) return 0; /* Uncompressible */
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
if (FSE_isError(errorCode)) return errorCode;
/* Write table description header */
- errorCode = FSE_writeNCount (op, FSE_MAX_HEADERSIZE, norm, maxSymbolValue, tableLog);
+ errorCode = FSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode;
op += errorCode;
/* Compress */
errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode;
- op += FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct);
+ errorCode = FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct);
+ if (errorCode == 0) return 0; /* not enough space for compressed data */
+ op += errorCode;
/* check compressibility */
if ( (size_t)(op-ostart) >= srcSize-1 )
*********************************************************/
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{
- U32* const base32 = (U32*)dt;
- FSE_decode_t* const cell = (FSE_decode_t*)(base32 + 1);
+ FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
+ FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
- base32[0] = 0;
+ DTableH->tableLog = 0;
+ DTableH->fastMode = 0;
cell->newState = 0;
cell->symbol = symbolValue;
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{
- U32* const base32 = (U32*)dt;
- FSE_decode_t* dinfo = (FSE_decode_t*)(base32 + 1);
+ FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
+ FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
if (nbBits < 1) return (size_t)-FSE_ERROR_GENERIC; /* min size */
/* Build Decoding Table */
- base32[0] = nbBits;
+ DTableH->tableLog = (U16)nbBits;
+ DTableH->fastMode = 1;
for (s=0; s<=maxSymbolValue; s++)
{
dinfo[s].newState = 0;
}
if (bitD->ptr == bitD->start)
{
- if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_partiallyFilled;
+ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_endOfBuffer;
if (bitD->bitsConsumed == sizeof(bitD->bitContainer)*8) return FSE_DStream_completed;
return FSE_DStream_tooFar;
}
if (bitD->ptr - nbBytes < bitD->start)
{
nbBytes = (U32)(bitD->ptr - bitD->start); /* note : necessarily ptr > start */
- result = FSE_DStream_partiallyFilled;
+ result = FSE_DStream_endOfBuffer;
}
bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes*8;
void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt)
{
- const U32* const base32 = (const U32*)dt;
- DStatePtr->state = FSE_readBits(bitD, base32[0]);
+ const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
+ DStatePtr->state = FSE_readBits(bitD, DTableH->tableLog);
FSE_reloadDStream(bitD);
- DStatePtr->table = base32 + 1;
+ DStatePtr->table = dt + 1;
}
BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD)
FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
- const FSE_DTable* dt, unsigned fast)
+ const FSE_DTable* dt, const unsigned fast)
{
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
- const FSE_DTable* dt, size_t fastMode)
+ const FSE_DTable* dt)
{
+ const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt;
+ const U32 fastMode = DTableH->fastMode;
+
/* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
- size_t errorCode, fastMode;
+ size_t errorCode;
if (cSrcSize<2) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */
ip += errorCode;
cSrcSize -= errorCode;
- fastMode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
- if (FSE_isError(fastMode)) return fastMode;
+ errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
+ if (FSE_isError(errorCode)) return errorCode;
/* always return, even if it is an error code */
- return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt, fastMode);
+ return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);
}
BYTE nbBits;
} nodeElt;
-
+/* HUF_writeCTable() :
+ return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32 maxSymbolValue, U32 huffLog)
{
+ BYTE bitsToWeight[HUF_ABSOLUTEMAX_TABLELOG + 1];
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
U32 n;
BYTE* op = (BYTE*)dst;
size_t size;
- // check conditions
+ /* check conditions */
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1)
return (size_t)-FSE_ERROR_GENERIC;
+ /* convert to weight */
+ bitsToWeight[0] = 0;
+ for (n=1; n<=huffLog; n++)
+ bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
for (n=0; n<maxSymbolValue; n++)
- huffWeight[n] = tree[n].nbBits ? (BYTE)(huffLog + 1 - tree[n].nbBits) : 0;
+ huffWeight[n] = bitsToWeight[tree[n].nbBits];
- size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); // don't need last symbol stat : implied
+ size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */
if (FSE_isError(size)) return size;
- if (size >= 128) return (size_t)-FSE_ERROR_GENERIC; // should never happen, since maxSymbolValue <= 255
+ if (size >= 128) return (size_t)-FSE_ERROR_GENERIC; /* should never happen, since maxSymbolValue <= 255 */
if ((size <= 1) || (size >= maxSymbolValue/2))
{
- if (maxSymbolValue > 64) return (size_t)-FSE_ERROR_GENERIC; // special case, not implemented (not possible)
- if (size==1) // RLE
+ if (size==1) /* RLE */
{
- op[0] = (BYTE)(128 /*special case*/ + 64 /* RLE */ + (maxSymbolValue-1));
- op[1] = huffWeight[0];
- return 2;
+ /* only possible case : serie of 1 (because there are at least 2) */
+ /* can only be 2^n or (2^n-1), otherwise not an huffman tree */
+ BYTE code;
+ switch(maxSymbolValue)
+ {
+ case 1: code = 0; break;
+ case 2: code = 1; break;
+ case 3: code = 2; break;
+ case 4: code = 3; break;
+ case 7: code = 4; break;
+ case 8: code = 5; break;
+ case 15: code = 6; break;
+ case 16: code = 7; break;
+ case 31: code = 8; break;
+ case 32: code = 9; break;
+ case 63: code = 10; break;
+ case 64: code = 11; break;
+ case 127: code = 12; break;
+ case 128: code = 13; break;
+ default : return (size_t)-FSE_ERROR_corruptionDetected;
+ }
+ op[0] = (BYTE)(255-13 + code);
+ return 1;
}
- // Not compressible
+ /* Not compressible */
+ if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC; /* not implemented (not possible with current format) */
+ if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
+ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
for (n=0; n<maxSymbolValue; n+=2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
return ((maxSymbolValue+1)/2) + 1;
}
- // normal case
+ /* normal header case */
op[0] = (BYTE)size;
return size+1;
}
int totalCost = 0;
const U32 largestBits = huffNode[lastNonNull].nbBits;
- // early exit : all is fine
+ /* early exit : all is fine */
if (largestBits <= maxNbBits) return largestBits;
// now we have a few too large elements (at least >= 2)
}
#define FSE_FLUSHBITS_1(stream) \
- if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) FSE_flushBits(stream)
+ if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) FSE_FLUSHBITS(stream)
#define FSE_FLUSHBITS_2(stream) \
- if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) FSE_flushBits(stream)
+ if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) FSE_FLUSHBITS(stream)
-static size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, HUF_CElt* CTable)
+size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, HUF_CElt* CTable)
{
const BYTE* ip = (const BYTE*) src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = (BYTE*) ostart;
+ BYTE* const oend = ostart + dstSize;
U16* jumpTable = (U16*) dst;
size_t n, streamSize;
+ const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
+ size_t errorCode;
FSE_CStream_t bitC;
/* init */
- (void)dstSize; /* objective : ensure it fits into dstBuffer (Todo) */
op += 6; /* jump Table -- could be optimized by delta / deviation */
- FSE_initCStream(&bitC, op);
+ errorCode = FSE_initCStream(&bitC, op, dstSize);
+ if (FSE_isError(errorCode)) return 0;
n = srcSize & ~15; // mod 16
switch (srcSize & 15)
case 13: HUF_encodeSymbol(&bitC, ip[n+12], CTable);
FSE_FLUSHBITS_1(&bitC);
case 12: HUF_encodeSymbol(&bitC, ip[n+11], CTable);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
case 11: HUF_encodeSymbol(&bitC, ip[n+10], CTable);
FSE_FLUSHBITS_1(&bitC);
case 10: HUF_encodeSymbol(&bitC, ip[n+ 9], CTable);
case 9 : HUF_encodeSymbol(&bitC, ip[n+ 8], CTable);
FSE_FLUSHBITS_1(&bitC);
case 8 : HUF_encodeSymbol(&bitC, ip[n+ 7], CTable);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
case 7 : HUF_encodeSymbol(&bitC, ip[n+ 6], CTable);
FSE_FLUSHBITS_1(&bitC);
case 6 : HUF_encodeSymbol(&bitC, ip[n+ 5], CTable);
case 5 : HUF_encodeSymbol(&bitC, ip[n+ 4], CTable);
FSE_FLUSHBITS_1(&bitC);
case 4 : HUF_encodeSymbol(&bitC, ip[n+ 3], CTable);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
FSE_FLUSHBITS_2(&bitC);
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
FSE_FLUSHBITS_1(&bitC);
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
case 0 :
default: ;
}
HUF_encodeSymbol(&bitC, ip[n-12], CTable);
FSE_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n-16], CTable);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
}
streamSize = FSE_closeCStream(&bitC);
+ if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
FSE_writeLE16(jumpTable, (U16)streamSize);
op += streamSize;
- FSE_initCStream(&bitC, op);
+ errorCode = FSE_initCStream(&bitC, op, oend-op);
+ if (FSE_isError(errorCode)) return 0;
n = srcSize & ~15; // mod 16
for (; n>0; n-=16)
{
HUF_encodeSymbol(&bitC, ip[n-11], CTable);
FSE_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n-15], CTable);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
}
streamSize = FSE_closeCStream(&bitC);
+ if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
FSE_writeLE16(jumpTable+1, (U16)streamSize);
op += streamSize;
- FSE_initCStream(&bitC, op);
+ errorCode = FSE_initCStream(&bitC, op, oend-op);
+ if (FSE_isError(errorCode)) return 0;
n = srcSize & ~15; // mod 16
for (; n>0; n-=16)
{
HUF_encodeSymbol(&bitC, ip[n-10], CTable);
FSE_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n-14], CTable);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
}
streamSize = FSE_closeCStream(&bitC);
+ if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
FSE_writeLE16(jumpTable+2, (U16)streamSize);
op += streamSize;
- FSE_initCStream(&bitC, op);
+ errorCode = FSE_initCStream(&bitC, op, oend-op);
+ if (FSE_isError(errorCode)) return 0;
n = srcSize & ~15; // mod 16
for (; n>0; n-=16)
{
HUF_encodeSymbol(&bitC, ip[n- 9], CTable);
FSE_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n-13], CTable);
- FSE_flushBits(&bitC);
+ FSE_FLUSHBITS(&bitC);
}
streamSize = FSE_closeCStream(&bitC);
+ if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
op += streamSize;
return op-ostart;
size_t errorCode;
/* early out */
- if (dstSize < FSE_compressBound(srcSize)) return (size_t)-FSE_ERROR_dstSize_tooSmall;
if (srcSize <= 1) return srcSize; /* Uncompressed or RLE */
if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
/* check compressibility */
if ((size_t)(op-ostart) >= srcSize-1)
- return 0;
+ return op-ostart;
return op-ostart;
}
size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
- U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1] = {0};
- U32 weightTotal = 0;
+ U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
+ U32 weightTotal;
U32 maxBits;
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
U32 nextRankStart;
HUF_DElt* const dt = (HUF_DElt*)(DTable + 1);
- FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); // if compilation fails here, assertion is false
- if (iSize >= 128) // special case
+ FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */
+ //memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */
+ if (iSize >= 128) /* special header */
{
- if (iSize >= (128+64)) // RLE
+ if (iSize >= (242)) /* RLE */
{
- if (srcSize < 2) return (size_t)-FSE_ERROR_srcSize_wrong;
- oSize = (iSize & 63) + 1;
- memset(huffWeight, ip[1], oSize);
- iSize = 1;
+ static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
+ oSize = l[iSize-242];
+ memset(huffWeight, 1, oSize);
+ iSize = 0;
}
- else // Incompressible
+ else /* Incompressible */
{
- oSize = (iSize & 63) + 1;
+ oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
ip += 1;
for (n=0; n<oSize; n+=2)
{
- huffWeight[n] = (ip[n/2] >> 4);
- huffWeight[n+1] = (ip[n/2] & 15);
+ huffWeight[n] = ip[n/2] >> 4;
+ huffWeight[n+1] = ip[n/2] & 15;
}
}
}
- else // normal case, header compressed with FSE
+ else /* header compressed with FSE (normal case) */
{
if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
- oSize = FSE_decompress(huffWeight, HUF_MAX_SYMBOL_VALUE, ip+1, iSize); // max 255 values stored, last is implied
+ oSize = FSE_decompress(huffWeight, HUF_MAX_SYMBOL_VALUE, ip+1, iSize); /* max 255 values decoded, last one is implied */
if (FSE_isError(oSize)) return oSize;
}
-
- // stats on weights
+ /* collect weight stats */
+ memset(rankVal, 0, sizeof(rankVal));
+ weightTotal = 0;
for (n=0; n<oSize; n++)
{
+ if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return (size_t)-FSE_ERROR_corruptionDetected;
rankVal[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
- // get last symbol weight(implied)
+ /* get last non-null symbol weight (implied, total must be 2^n) */
maxBits = FSE_highbit32(weightTotal) + 1;
- if (maxBits > DTable[0]) return (size_t)-FSE_ERROR_GENERIC; // DTable is too small
+ if (maxBits > DTable[0]) return (size_t)-FSE_ERROR_tableLog_tooLarge; /* DTable is too small */
DTable[0] = (U16)maxBits;
{
U32 total = 1 << maxBits;
U32 rest = total - weightTotal;
U32 verif = 1 << FSE_highbit32(rest);
- if (verif != rest) return (size_t)-FSE_ERROR_GENERIC; // last value must be a clean power of 2
- huffWeight[oSize] = (BYTE)(FSE_highbit32(rest) + 1);
- rankVal[huffWeight[oSize]]++;
+ U32 lastWeight = FSE_highbit32(rest) + 1;
+ if (verif != rest) return (size_t)-FSE_ERROR_corruptionDetected; /* last value must be a clean power of 2 */
+ huffWeight[oSize] = (BYTE)lastWeight;
+ rankVal[lastWeight]++;
}
- // Prepare ranks
+ /* check tree construction validity */
+ if ((rankVal[1] < 2) || (rankVal[1] & 1)) return (size_t)-FSE_ERROR_corruptionDetected; /* by construction : at least 2 elts of rank 1, must be even */
+
+ /* Prepare ranks */
nextRankStart = 0;
for (n=1; n<=maxBits; n++)
{
rankVal[n] = current;
}
- // fill table
+ /* fill DTable */
for (n=0; n<=oSize; n++)
{
- U32 i;
const U32 w = huffWeight[n];
const U32 length = (1 << w) >> 1;
+ U32 i;
HUF_DElt D;
D.byte = (BYTE)n; D.nbBits = (BYTE)(maxBits + 1 - w);
for (i = rankVal[w]; i < rankVal[w] + length; i++)
return iSize+1;
}
-static void HUF_decodeSymbol(BYTE* ptr, FSE_DStream_t* Dstream, const HUF_DElt* dt, U32 dtLog)
+
+static BYTE HUF_decodeSymbol(FSE_DStream_t* Dstream, const HUF_DElt* dt, const U32 dtLog)
{
- size_t val = FSE_lookBitsFast(Dstream, dtLog);
- BYTE c = dt[val].byte;
+ const size_t val = FSE_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
+ const BYTE c = dt[val].byte;
FSE_skipBits(Dstream, dt[val].nbBits);
- *ptr = c;
+ return c;
}
-static size_t HUF_decompress_usingDTable(
+static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
const U16* DTable)
const char* const start4 = start3 + length3;
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
+ if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
+
errorCode = FSE_initDStream(&bitD1, start1, length1);
if (FSE_isError(errorCode)) return errorCode;
errorCode = FSE_initDStream(&bitD2, start2, length2);
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
{
#define HUF_DECODE_SYMBOL_0(n, Dstream) \
- HUF_decodeSymbol(op+n, &Dstream, dt, dtLog);
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
#define HUF_DECODE_SYMBOL_1(n, Dstream) \
- HUF_decodeSymbol(op+n, &Dstream, dt, dtLog); \
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
#define HUF_DECODE_SYMBOL_2(n, Dstream) \
- HUF_decodeSymbol(op+n, &Dstream, dt, dtLog); \
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
if (FSE_32bits()) FSE_reloadDStream(&Dstream)
HUF_DECODE_SYMBOL_1( 0, bitD1);
HUF_DECODE_SYMBOL_0(15, bitD4);
}
- if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be 0 (unfinished) */
+ if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
return (size_t)-FSE_ERROR_corruptionDetected;
/* tail */
FSE_DStream_t bitTail;
bitTail.ptr = bitD1.ptr;
bitTail.bitsConsumed = bitD1.bitsConsumed;
- bitTail.bitContainer = bitD1.bitContainer; // required in case FSE_DStream_partiallyFilled
+ bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
bitTail.start = start1;
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
{
/*
FSE_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
- 'dst' buffer must be already allocated, and sized to handle worst case situations.
- Worst case size evaluation is provided by FSE_compressBound().
- return : size of compressed data
- Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
- if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
- if FSE_isError(return), it's an error code.
+ 'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize)
+ return : size of compressed data (<= maxDstSize)
+ Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
+ if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
+ if FSE_isError(return), compression failed (more details using FSE_getErrorName())
FSE_decompress():
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
/******************************************
* Huff0 simple functions
******************************************/
-size_t HUF_compress (void* dst, size_t dstSize, const void* src, size_t srcSize);
+size_t HUF_compress(void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize);
size_t HUF_decompress(void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize);
/*
HUF_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
- 'dst' buffer must be already allocated, and sized to handle worst case situations.
- Worst case size evaluation is provided by FSE_compressBound().
- return : size of compressed data
- Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
+ 'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= HUF_compressBound(srcSize)
+ return : size of compressed data (<= maxDstSize)
+ Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
- if FSE_isError(return), it's an error code.
+ if FSE_isError(return), compression failed (more details using FSE_getErrorName())
HUF_decompress():
Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize',
FSE_compress() does the following:
1. count symbol occurrence from source[] into table count[]
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
-3. save normalized counters to memory buffer using writeHeader()
+3. save normalized counters to memory buffer using writeNCount()
4. build encoding table 'CTable' from normalized counters
5. encode the data stream using encoding table 'CTable'
FSE_decompress() does the following:
-1. read normalized counters with readHeader()
+1. read normalized counters with readNCount()
2. build decoding table 'DTable' from normalized counters
3. decode the data stream using decoding table 'DTable'
-The following API allows to trigger specific sub-functions for advanced tasks.
+The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
-or to save and provide normalized distribution using one's own method.
+or to save and provide normalized distribution using external method.
*/
/* *** COMPRESSION *** */
/*
Constructor and Destructor of type FSE_CTable
-Not that its size depends on parameters 'tableLog' and 'maxSymbolValue' */
-typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void */
+ Note that its size depends on 'tableLog' and 'maxSymbolValue' */
+typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
void FSE_freeCTable (FSE_CTable* ct);
Builds 'ct', which must be already allocated, using FSE_createCTable()
return : 0
or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
FSE_compress_usingCTable():
Compress 'src' using 'ct' into 'dst' which must be already allocated
- return : size of compressed data
+ return : size of compressed data (<= maxDstSize)
+ or 0 if compressed data could not fit into 'dst'
or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_compress_usingCTable (void* dst, size_t dstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
+size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
/*
Tutorial :
----------
-The first step is to count all symbols. FSE_count() provides one quick way to do this job.
+The first step is to count all symbols. FSE_count() does this job very fast.
Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
FSE_count() will return the number of occurrence of the most frequent symbol.
+This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
The next step is to normalize the frequencies.
FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
-It also guarantees a minimum of 1 to any Symbol which frequency is >= 1.
-You can use input 'tableLog'==0 to mean "use default tableLog value".
-If you are unsure of which tableLog value to use, you can optionally call FSE_optimalTableLog(),
+It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
+You can use 'tableLog'==0 to mean "use default tableLog value".
+If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
The result of FSE_normalizeCount() will be saved into a table,
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
The return value is tableLog if everything proceeded as expected.
It is 0 if there is a single symbol within distribution.
-If there is an error(typically, invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
+If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
-'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeHeader().
-'header' buffer must be already allocated.
+'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
+'buffer' must be already allocated.
For guaranteed success, buffer size must be at least FSE_headerBound().
-The result of the function is the number of bytes written into 'header'.
-If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()) (for example, buffer size too small).
+The result of the function is the number of bytes written into 'buffer'.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
'normalizedCounter' can then be used to create the compression table 'CTable'.
-The space required by 'CTable' must be already allocated. Its size is provided by FSE_sizeof_CTable().
-'CTable' must be aligned of 4 bytes boundaries.
+The space required by 'CTable' must be already allocated, using FSE_createCTable().
You can then use FSE_buildCTable() to fill 'CTable'.
-In both cases, if there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
+If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
-The function returns the size of compressed data (without header).
+The function returns the size of compressed data (without header), necessarily <= maxDstSize.
+If it returns '0', compressed data could not fit into 'dst'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
*/
/*
Constructor and Destructor of type FSE_DTable
-Note that its size depends on parameters 'tableLog' */
-typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void */
+ Note that its size depends on 'tableLog' */
+typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_DTable* FSE_createDTable(unsigned tableLog);
void FSE_freeDTable(FSE_DTable* dt);
/*
FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable()
- return : 1 if 'dt' is compatible with fast mode, 0 otherwise,
+ return : 0,
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
FSE_decompress_usingDTable():
- Decompress compressed source 'cSrc' of size 'cSrcSize'
- using 'dt' into 'dst' which must be already allocated.
- Use fastMode==1 only if authorized by result of FSE_buildDTable().
+ Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt'
+ into 'dst' which must be already allocated.
return : size of regenerated data (necessarily <= maxDstSize)
or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt, size_t fastMode);
+size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
/*
Tutorial :
If block is a single repeated byte, use memset() instead )
The first step is to obtain the normalized frequencies of symbols.
-This can be performed by reading a header with FSE_readHeader().
-'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of short.
+This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
or size the table to handle worst case situations (typically 256).
-FSE_readHeader will provide 'tableLog' and 'maxSymbolValue' stored into the header.
-The result of FSE_readHeader() is the number of bytes read from 'header'.
-Note that 'headerSize' must be at least 4 bytes, even if useful information is less than that.
+FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
+The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
+Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
If there is an error, the function will return an error code, which can be tested using FSE_isError().
-The next step is to create the decompression tables 'FSE_DTable' from 'normalizedCounter'.
+The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
This is performed by the function FSE_buildDTable().
The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
-The function will return 1 if FSE_DTable is compatible with fastMode, 0 otherwise.
If there is an error, the function will return an error code, which can be tested using FSE_isError().
'FSE_DTable' can then be used to decompress 'cSrc', with FSE_decompress_usingDTable().
-Only trigger fastMode if it was authorized by the result of FSE_buildDTable(), otherwise decompression will fail.
-cSrcSize must be correct, otherwise decompression will fail.
-FSE_decompress_usingDTable() result will tell how many bytes were regenerated.
-If there is an error, the function will return an error code, which can be tested using FSE_isError().
+'cSrcSize' must be strictly correct, otherwise decompression will fail.
+FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=maxDstSize).
+If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
*/
/******************************************
* Static allocation
******************************************/
-#define FSE_MAX_HEADERSIZE 512
-#define FSE_COMPRESSBOUND(size) (size + (size>>7) + FSE_MAX_HEADERSIZE) /* Macro can be useful for static allocation */
+/* FSE buffer bounds */
+#define FSE_NCOUNTBOUND 512
+#define FSE_BLOCKBOUND(size) (size + (size>>7))
+#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
-/* You can statically allocate CTable/DTable as a table of unsigned using below macro */
+/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
-/* You can statically allocate a Huff0 DTable as a table of unsigned char using below macro */
+/* Huff0 buffer bounds */
+#define HUF_CTABLEBOUND 129
+#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if pre-filtered with fast heuristic */
+#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
+
+/* You can statically allocate Huff0 DTable as a table of unsigned short using below macro */
#define HUF_DTABLE_SIZE_U16(maxTableLog) (1 + (1<<maxTableLog))
#define HUF_CREATE_STATIC_DTABLE(DTable, maxTableLog) \
unsigned short DTable[HUF_DTABLE_SIZE_U16(maxTableLog)] = { maxTableLog }
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
Visual seems to do it automatically.
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
+ If none of these solutions is applicable, include "fse.c" directly.
*/
typedef struct
int bitPos;
char* startPtr;
char* ptr;
+ char* endPtr;
} FSE_CStream_t;
typedef struct
unsigned stateLog;
} FSE_CState_t;
-void FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer);
+size_t FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer, size_t maxDstSize);
void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
You will need a few variables to track your CStream. They are :
-FSE_CTable ct; // Provided by FSE_buildCTable()
-FSE_CStream_t bitC; // bitStream tracking structure
-FSE_CState_t state; // State tracking structure (can have several)
+FSE_CTable ct; // Provided by FSE_buildCTable()
+FSE_CStream_t bitStream; // bitStream tracking structure
+FSE_CState_t state; // State tracking structure (can have several)
The first thing to do is to init bitStream and state.
- FSE_initCStream(&bitC, dstBuffer);
+ size_t errorCode = FSE_initCStream(&bitStream, dstBuffer, maxDstSize);
FSE_initCState(&state, ct);
+Note that FSE_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
You can then encode your input data, byte after byte.
-FSE_encodeByte() outputs a maximum of 'tableLog' bits at a time.
+FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
Remember decoding will be done in reverse direction.
FSE_encodeByte(&bitStream, &state, symbol);
Your last FSE encoding operation shall be to flush your last state value(s).
FSE_flushState(&bitStream, &state);
-Finally, you must then close the bitStream.
-The function returns the size in bytes of CStream.
+Finally, you must close the bitStream.
+The function returns the size of CStream in bytes.
+If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
size_t size = FSE_closeCStream(&bitStream);
*/
unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
typedef enum { FSE_DStream_unfinished = 0,
- FSE_DStream_partiallyFilled = 1,
+ FSE_DStream_endOfBuffer = 1,
FSE_DStream_completed = 2,
FSE_DStream_tooFar = 3 } FSE_DStream_status; /* result of FSE_reloadDStream() */
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... ?! */
You will need a few variables to track your bitStream. They are :
-FSE_DStream_t DStream; // Stream context
-FSE_DState_t DState; // State context. Multiple ones are possible
-FSE_DTable dt; // Decoding table, provided by FSE_buildDTable()
-U32 tableLog; // Provided by FSE_readHeader()
+FSE_DStream_t DStream; // Stream context
+FSE_DState_t DState; // State context. Multiple ones are possible
+FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
The first thing to do is to init the bitStream.
- errorCode = FSE_initDStream(&DStream, &optionalId, srcBuffer, srcSize);
+ errorCode = FSE_initDStream(&DStream, srcBuffer, srcSize);
-You should then retrieve your initial state(s) :
- errorCode = FSE_initDState(&DState, &DStream, dt, tableLog);
+You should then retrieve your initial state(s)
+(in reverse flushing order if you have several ones) :
+ errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
You can then decode your data, symbol after symbol.
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
FSE_reloadDStream() result tells if there is still some more data to read from DStream.
FSE_DStream_unfinished : there is still some data left into the DStream.
-FSE_DStream_partiallyFilled : Dstream reached end of buffer. Its container may no longer be completely filled.
+FSE_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
FSE_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
FSE_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
-When reaching end of buffer (FSE_DStream_partiallyFilled), progress slowly, notably if you decode multiple symbols per loop,
+When reaching end of buffer (FSE_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
to properly detect the exact end of stream.
After each decoded symbol, check if DStream is fully consumed using this simple test :
FSE_reloadDStream(&DStream) >= FSE_DStream_completed
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
Checking if DStream has reached its end is performed by :
FSE_endOfDStream(&DStream);
-Check also the states. There might be some entropy left there, able to decode some high probability (>50%) symbol.
+Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
FSE_endOfDState(&DState);
*/
/* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */
unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD);
-/* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */
+/* faster, but works only if allways nbBits >= 1 (otherwise, result will be corrupted) */
#if defined (__cplusplus)
}
-/* return : size of CStream in bits */
-size_t ZSTD_compressLiterals_usingCTable(void* dst, size_t dstSize,
- const void* src, size_t srcSize,
- const FSE_CTable* CTable)
-{
- const BYTE* const istart = (const BYTE*)src;
- const BYTE* ip = istart;
- const BYTE* const iend = istart + srcSize;
- FSE_CStream_t bitC;
- FSE_CState_t CState1, CState2;
-
- /* init */
- (void)dstSize; // objective : ensure it fits into dstBuffer (Todo)
- FSE_initCStream(&bitC, dst);
- FSE_initCState(&CState1, CTable);
- CState2 = CState1;
-
- /* Note : at this stage, srcSize > LITERALS_NOENTROPY (checked by ZSTD_compressLiterals()) */
- // join to mod 2
- if (srcSize & 1)
- {
- FSE_encodeSymbol(&bitC, &CState1, *ip++);
- FSE_flushBits(&bitC);
- }
-
- // join to mod 4
- if ((sizeof(size_t)*8 > LitFSELog*4+7 ) && (srcSize & 2)) // test bit 2
- {
- FSE_encodeSymbol(&bitC, &CState2, *ip++);
- FSE_encodeSymbol(&bitC, &CState1, *ip++);
- FSE_flushBits(&bitC);
- }
-
- // 2 or 4 encoding per loop
- while (ip<iend)
- {
- FSE_encodeSymbol(&bitC, &CState2, *ip++);
-
- if (sizeof(size_t)*8 < LitFSELog*2+7 ) // this test must be static
- FSE_flushBits(&bitC);
-
- FSE_encodeSymbol(&bitC, &CState1, *ip++);
-
- if (sizeof(size_t)*8 > LitFSELog*4+7 ) // this test must be static
- {
- FSE_encodeSymbol(&bitC, &CState2, *ip++);
- FSE_encodeSymbol(&bitC, &CState1, *ip++);
- }
-
- FSE_flushBits(&bitC);
- }
-
- FSE_flushCState(&bitC, &CState2);
- FSE_flushCState(&bitC, &CState1);
- return FSE_closeCStream(&bitC);
-}
-
-
size_t ZSTD_minGain(size_t srcSize)
{
return (srcSize >> 6) + 1;
const seqStore_t* seqStorePtr,
size_t lastLLSize, size_t srcSize)
{
- FSE_CStream_t blockStream;
U32 count[256];
S16 norm[256];
size_t mostFrequent;
const U32* op_offset = seqStorePtr->offset;
const BYTE* op_matchLength = seqStorePtr->matchLength;
const size_t nbSeq = op_litLength - op_litLength_start;
- BYTE* op;
- BYTE* offsetBits_start = seqStorePtr->offCodeStart;
+ BYTE* op = dst;
+ BYTE* const oend = dst + maxDstSize;
+ BYTE* const offsetBits_start = seqStorePtr->offCodeStart;
BYTE* offsetBitsPtr = offsetBits_start;
const size_t minGain = ZSTD_minGain(srcSize);
const size_t maxCSize = srcSize - minGain;
const size_t maxLSize = maxCSize > minSeqSize ? maxCSize - minSeqSize : 0;
BYTE* seqHead;
- /* init */
- op = dst;
- /* Encode literals */
+ /* Compress literals */
{
size_t cSize;
size_t litSize = op_lit - op_lit_start;
op += dumpsLength;
}
- /* Encoding table of Literal Lengths */
+ /* CTable for Literal Lengths */
max = MaxLL;
mostFrequent = FSE_countFast(count, &max, seqStorePtr->litLengthStart, nbSeq);
if ((mostFrequent == nbSeq) && (nbSeq > 2))
{
tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
- op += FSE_writeNCount(op, maxDstSize, norm, max, tableLog);
+ op += FSE_writeNCount(op, oend-op, norm, max, tableLog);
FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
LLtype = bt_compressed;
}
- /* Encoding table of Offsets */
+ /* CTable for Offsets codes */
{
- /* create OffsetBits */
+ /* create Offset codes */
size_t i;
const U32* const op_offset_start = seqStorePtr->offsetStart;
max = MaxOff;
{
tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
- op += FSE_writeNCount(op, maxDstSize, norm, max, tableLog);
+ op += FSE_writeNCount(op, oend-op, norm, max, tableLog);
FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
Offtype = bt_compressed;
}
- /* Encoding Table of MatchLengths */
+ /* CTable for MatchLengths */
max = MaxML;
mostFrequent = FSE_countFast(count, &max, seqStorePtr->matchLengthStart, nbSeq);
if ((mostFrequent == nbSeq) && (nbSeq > 2))
{
tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
- op += FSE_writeNCount(op, maxDstSize, norm, max, tableLog);
+ op += FSE_writeNCount(op, oend-op, norm, max, tableLog);
FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
MLtype = bt_compressed;
}
seqHead[0] += (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
- /* Encoding */
+ /* Encoding Sequences */
{
+ size_t streamSize, errorCode;
+ FSE_CStream_t blockStream;
FSE_CState_t stateMatchLength;
FSE_CState_t stateOffsetBits;
FSE_CState_t stateLitLength;
- FSE_initCStream(&blockStream, op);
+ errorCode = FSE_initCStream(&blockStream, op, oend-op);
+ if (FSE_isError(errorCode)) return 0; /* not enough space remaining */
FSE_initCState(&stateMatchLength, CTable_MatchLength);
FSE_initCState(&stateOffsetBits, CTable_OffsetBits);
FSE_initCState(&stateLitLength, CTable_LitLength);
FSE_flushCState(&blockStream, &stateMatchLength);
FSE_flushCState(&blockStream, &stateOffsetBits);
FSE_flushCState(&blockStream, &stateLitLength);
- }
- op += FSE_closeCStream(&blockStream);
+ streamSize = FSE_closeCStream(&blockStream);
+ if (streamSize==0) return 0; /* not enough space */
+ op += streamSize;
+ }
/* check compressibility */
if ((size_t)(op-dst) >= maxCSize) return 0;
projects / thirdparty / zstd.git / commitdiff
