#define WORKPLACESIZE (BLOCKSIZE*3)
#define MINMATCH 4
-#define MLbits 7
-#define LLbits 6
-#define Offbits 5
-#define MaxML ((1<<MLbits )-1)
-#define MaxLL ((1<<LLbits )-1)
-#define MaxOff 31
#define LitFSELog 11
#define MLFSELog 10
#define LLFSELog 10
/* *******************************************************
* Memory operations
**********************************************************/
-static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
-
-static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
-
-#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
-
-/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
-static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
-{
- const BYTE* ip = (const BYTE*)src;
- BYTE* op = (BYTE*)dst;
- BYTE* const oend = op + length;
- do COPY8(op, ip) while (op < oend);
-}
+static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
/* **************************************
* Local structures
****************************************/
-typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
-
-typedef struct
-{
- blockType_t blockType;
- U32 origSize;
-} blockProperties_t;
-
-typedef struct {
- void* buffer;
- U32* offsetStart;
- U32* offset;
- BYTE* offCodeStart;
- BYTE* offCode;
- BYTE* litStart;
- BYTE* lit;
- BYTE* litLengthStart;
- BYTE* litLength;
- BYTE* matchLengthStart;
- BYTE* matchLength;
- BYTE* dumpsStart;
- BYTE* dumps;
-} seqStore_t;
-
void ZSTD_resetSeqStore(seqStore_t* ssPtr)
{
ssPtr->offset = ssPtr->offsetStart;
ssPtr->dumps = ssPtr->dumpsStart;
}
-
struct ZSTD_CCtx_s
{
const BYTE* base;
ctx->seqStore.litLengthStart = ctx->seqStore.litStart + BLOCKSIZE;
ctx->seqStore.matchLengthStart = ctx->seqStore.litLengthStart + (BLOCKSIZE>>2);
ctx->seqStore.dumpsStart = ctx->seqStore.matchLengthStart + (BLOCKSIZE>>2);
- memset(ctx->hashTable, 0, HASH_TABLESIZE*4);
+ memset(ctx->hashTable, 0, sizeof(ctx->hashTable));
}
ZSTD_CCtx* ZSTD_createCCtx(void)
}
-/* *************************************
-* Function body to include
-***************************************/
-#include "mem.h"
-static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
-
-MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
-{
- if (MEM_isLittleEndian())
- {
- if (MEM_64bits())
- {
-# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanForward64( &r, (U64)val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctzll((U64)val) >> 3);
-# else
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
- return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-# endif
- }
- else /* 32 bits */
- {
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r;
- _BitScanForward( &r, (U32)val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctz((U32)val) >> 3);
-# else
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-# endif
- }
- }
- else /* Big Endian CPU */
- {
- if (MEM_32bits())
- {
-# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanReverse64( &r, val );
- return (unsigned)(r>>3);
-# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clzll(val) >> 3);
-# else
- unsigned r;
- const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
- if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
- r += (!val);
- return r;
-# endif
- }
- else /* 32 bits */
- {
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanReverse( &r, (unsigned long)val );
- return (unsigned)(r>>3);
-# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clz((U32)val) >> 3);
-# else
- unsigned r;
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
- r += (!val);
- return r;
-# endif
- }
- }
-}
-
-
-MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
-{
- const BYTE* const pStart = pIn;
-
- while ((pIn<pInLimit-(sizeof(size_t)-1)))
- {
- size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
- if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
- pIn += ZSTD_NbCommonBytes(diff);
- return (size_t)(pIn - pStart);
- }
-
- if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
- if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
- if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
- return (size_t)(pIn - pStart);
-}
-
-
/* *******************************************************
* Compression
*********************************************************/
}
-static size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
BYTE* const ostart = (BYTE* const)dst;
size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 1; }
static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
- const void* src, size_t srcSize)
+ const void* src, size_t srcSize)
{
const size_t minGain = ZSTD_minGain(srcSize);
BYTE* const ostart = (BYTE*)dst;
}
-static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
- const seqStore_t* seqStorePtr,
- size_t srcSize)
+size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
+ const seqStore_t* seqStorePtr,
+ size_t srcSize)
{
U32 count[MaxSeq+1];
S16 norm[MaxSeq+1];
U32 CTable_MatchLength[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML )];
U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
const BYTE* const op_lit_start = seqStorePtr->litStart;
- const BYTE* op_lit = seqStorePtr->lit;
const BYTE* const llTable = seqStorePtr->litLengthStart;
const BYTE* const llPtr = seqStorePtr->litLength;
const BYTE* const mlTable = seqStorePtr->matchLengthStart;
/* Compress literals */
{
size_t cSize;
- size_t litSize = op_lit - op_lit_start;
+ size_t litSize = seqStorePtr->lit - op_lit_start;
if (litSize <= LITERAL_NOENTROPY)
cSize = ZSTD_compressRawLiteralsBlock(op, maxDstSize, op_lit_start, litSize);
}
-static void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
-{
- BYTE* op_lit = seqStorePtr->lit;
- BYTE* const l_end = op_lit + litLength;
-
- /* copy Literals */
- while (op_lit<l_end) COPY8(op_lit, literals);
- seqStorePtr->lit += litLength;
-
- /* literal Length */
- if (litLength >= MaxLL)
- {
- *(seqStorePtr->litLength++) = MaxLL;
- if (litLength<255 + MaxLL)
- *(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
- else
- {
- *(seqStorePtr->dumps++) = 255;
- MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
- }
- }
- else *(seqStorePtr->litLength++) = (BYTE)litLength;
-
- /* match offset */
- *(seqStorePtr->offset++) = (U32)offset;
-
- /* match Length */
- if (matchLength >= MaxML)
- {
- *(seqStorePtr->matchLength++) = MaxML;
- if (matchLength < 255+MaxML)
- *(seqStorePtr->dumps++) = (BYTE)(matchLength - MaxML);
- else
- {
- *(seqStorePtr->dumps++) = 255;
- MEM_writeLE32(seqStorePtr->dumps, (U32)matchLength); seqStorePtr->dumps+=3;
- }
- }
- else *(seqStorePtr->matchLength++) = (BYTE)matchLength;
-}
-
-
//static const U32 hashMask = (1<<HASH_LOG)-1;
//static const U64 prime5bytes = 889523592379ULL;
//static const U64 prime6bytes = 227718039650203ULL;
const BYTE* ip = istart + 1;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
- const BYTE* const ilimit = iend - 16;
+ const BYTE* const ilimit = iend - 8;
- size_t prevOffset=0, offset=0;
+ size_t prevOffset=4, offset=4;
/* init */
switch(*istart & 3)
{
- default:
case 0:
{
size_t nbLiterals = BLOCKSIZE;
dctx->litSize = litSize;
return 4;
}
+ default:
+ return ERROR(corruption_detected); /* forbidden nominal case */
}
}
const BYTE* match = op - sequence.offset;
/* check */
- if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */
//if (match > op) return ERROR(corruption_detected); /* address space overflow test (is clang optimizer removing this test ?) */
+ if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */
if (match < base) return ERROR(corruption_detected);
/* close range match, overlap */
seqState_t seqState;
memset(&sequence, 0, sizeof(sequence));
+ sequence.offset = 4;
seqState.dumps = dumps;
seqState.dumpsEnd = dumps + dumpsLength;
- seqState.prevOffset = 1;
+ seqState.prevOffset = 4;
errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
if (ERR_isError(errorCode)) return ERROR(corruption_detected);
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
size_t lastLLSize = litEnd - litPtr;
if (litPtr > litEnd) return ERROR(corruption_detected);
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
+ if (op != litPtr) memcpy(op, litPtr, lastLLSize);
op += lastLLSize;
}
}
}
-/*******************************
+/* ******************************
* Streaming Decompression API
-*******************************/
+********************************/
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
{
#include "error.h"
+/* *************************************
+* Function body to include
+***************************************/
+#include "mem.h"
+static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
+
+MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
+{
+ if (MEM_isLittleEndian())
+ {
+ if (MEM_64bits())
+ {
+# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r = 0;
+ _BitScanForward64( &r, (U64)val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_ctzll((U64)val) >> 3);
+# else
+ static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
+ return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+# endif
+ }
+ else /* 32 bits */
+ {
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r;
+ _BitScanForward( &r, (U32)val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_ctz((U32)val) >> 3);
+# else
+ static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
+ return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+# endif
+ }
+ }
+ else /* Big Endian CPU */
+ {
+ if (MEM_32bits())
+ {
+# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r = 0;
+ _BitScanReverse64( &r, val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_clzll(val) >> 3);
+# else
+ unsigned r;
+ const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
+ if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
+ if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+ r += (!val);
+ return r;
+# endif
+ }
+ else /* 32 bits */
+ {
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r = 0;
+ _BitScanReverse( &r, (unsigned long)val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_clz((U32)val) >> 3);
+# else
+ unsigned r;
+ if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+ r += (!val);
+ return r;
+# endif
+ }
+ }
+}
+
+
+MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
+{
+ const BYTE* const pStart = pIn;
+
+ while ((pIn<pInLimit-(sizeof(size_t)-1)))
+ {
+ size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
+ if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
+ pIn += ZSTD_NbCommonBytes(diff);
+ return (size_t)(pIn - pStart);
+ }
+
+ if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
+ if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
+ if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
+ return (size_t)(pIn - pStart);
+}
+
+
+static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
+
+#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
+
+/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
+static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
+{
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + length;
+ do COPY8(op, ip) while (op < oend);
+}
+
+
+typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
+
+typedef struct
+{
+ blockType_t blockType;
+ U32 origSize;
+} blockProperties_t;
+
+size_t ZSTD_noCompressBlock(void* op, size_t maxDstSize, const void* ip, size_t blockSize);
+
+
+typedef struct {
+ void* buffer;
+ U32* offsetStart;
+ U32* offset;
+ BYTE* offCodeStart;
+ BYTE* offCode;
+ BYTE* litStart;
+ BYTE* lit;
+ BYTE* litLengthStart;
+ BYTE* litLength;
+ BYTE* matchLengthStart;
+ BYTE* matchLength;
+ BYTE* dumpsStart;
+ BYTE* dumps;
+} seqStore_t;
+
+void ZSTD_resetSeqStore(seqStore_t* ssPtr);
+
+#define MLbits 7
+#define LLbits 6
+#define Offbits 5
+#define MaxML ((1<<MLbits) - 1)
+#define MaxLL ((1<<LLbits) - 1)
+#define MaxOff 31
+
+/** ZSTD_storeSeq
+ Store a sequence (literal length, literals, offset code and match length) into seqStore_t
+ @offsetCode : distance to match, or 0 == repCode
+ @matchCode : matchLength - MINMATCH
+*/
+MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
+{
+ /* copy Literals */
+ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
+ seqStorePtr->lit += litLength;
+
+ /* literal Length */
+ if (litLength >= MaxLL)
+ {
+ *(seqStorePtr->litLength++) = MaxLL;
+ if (litLength<255 + MaxLL)
+ *(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
+ else
+ {
+ *(seqStorePtr->dumps++) = 255;
+ MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
+ }
+ }
+ else *(seqStorePtr->litLength++) = (BYTE)litLength;
+
+ /* match offset */
+ *(seqStorePtr->offset++) = (U32)offsetCode;
+
+ /* match Length */
+ if (matchCode >= MaxML)
+ {
+ *(seqStorePtr->matchLength++) = MaxML;
+ if (matchCode < 255+MaxML)
+ *(seqStorePtr->dumps++) = (BYTE)(matchCode - MaxML);
+ else
+ {
+ *(seqStorePtr->dumps++) = 255;
+ MEM_writeLE32(seqStorePtr->dumps, (U32)matchCode); seqStorePtr->dumps += 3;
+ }
+ }
+ else *(seqStorePtr->matchLength++) = (BYTE)matchCode;
+}
+
+size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, const seqStore_t* seqStorePtr, size_t srcSize);
+
#if defined (__cplusplus)
}
#endif
--- /dev/null
+/*
+ ZSTD HC - High Compression Mode of Zstandard
+ Copyright (C) 2015, Yann Collet.
+
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - Zstd source repository : https://www.zstd.net
+*/
+
+
+/* *************************************
+* Includes
+***************************************/
+#include <stdlib.h> /* malloc */
+#include <string.h> /* memset */
+#include "zstdhc.h"
+#include "zstd_static.h"
+#include "mem.h"
+
+
+/* *************************************
+* Tuning Parameter
+***************************************/
+static const U32 ZSTD_HC_compressionLevel_default = 9;
+
+
+/* *************************************
+* Local Constants
+***************************************/
+#define MINMATCH 4
+#define MAX_DISTANCE 65535 /* <=== To be changed (dynamic ?) */
+
+#define DICTIONARY_LOGSIZE 16
+#define MAXD (1<<DICTIONARY_LOGSIZE)
+#define MAXD_MASK (MAXD - 1)
+
+#define HASH_LOG (DICTIONARY_LOGSIZE-1)
+#define HASHTABLESIZE (1 << HASH_LOG)
+#define HASH_MASK (HASHTABLESIZE - 1)
+
+static const U32 g_maxCompressionLevel = 19;
+
+#define KB *1024
+#define MB *1024*1024
+#define GB *(1ULL << 30)
+
+/* *************************************
+* Local Types
+***************************************/
+#define BLOCKSIZE (128 KB) /* define, for static allocation */
+#define WORKPLACESIZE (BLOCKSIZE*3)
+
+struct ZSTD_HC_CCtx_s
+{
+ U32 hashTable[HASHTABLESIZE];
+ U16 chainTable[MAXD];
+ const BYTE* end; /* next block here to continue on current prefix */
+ const BYTE* base; /* All index relative to this position */
+ const BYTE* dictBase; /* alternate base for extDict */
+ BYTE* inputBuffer; /* deprecated */
+ U32 dictLimit; /* below that point, need extDict */
+ U32 lowLimit; /* below that point, no more dict */
+ U32 nextToUpdate; /* index from which to continue dictionary update */
+ U32 compressionLevel;
+ seqStore_t seqStore;
+ BYTE buffer[WORKPLACESIZE];
+};
+
+
+ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void)
+{
+ return (ZSTD_HC_CCtx*) malloc(sizeof(ZSTD_HC_CCtx));
+}
+
+size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx) { free(cctx); return 0; }
+
+static void ZSTD_HC_resetCCtx (ZSTD_HC_CCtx* zc, U32 compressionLevel, const BYTE* start)
+{
+ if (compressionLevel==0) compressionLevel = ZSTD_HC_compressionLevel_default;
+ if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel;
+ memset(zc->hashTable, 0, sizeof(zc->hashTable));
+ memset(zc->chainTable, 0xFF, sizeof(zc->chainTable));
+ zc->nextToUpdate = 64 KB;
+ zc->base = start - 64 KB;
+ zc->end = start;
+ zc->dictBase = start - 64 KB;
+ zc->dictLimit = 64 KB;
+ zc->lowLimit = 64 KB;
+ zc->compressionLevel = compressionLevel;
+ zc->seqStore.buffer = zc->buffer;
+ zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
+ zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (BLOCKSIZE>>2));
+ zc->seqStore.litStart = zc->seqStore.offCodeStart + (BLOCKSIZE>>2);
+ zc->seqStore.litLengthStart = zc->seqStore.litStart + BLOCKSIZE;
+ zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (BLOCKSIZE>>2);
+ zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (BLOCKSIZE>>2);
+}
+
+/* *************************************
+* Local Macros
+***************************************/
+#define HASH_FUNCTION(u) (((u) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
+//#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */
+#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
+
+static U32 ZSTD_HC_hashPtr(const void* ptr) { return HASH_FUNCTION(MEM_read32(ptr)); }
+
+
+/* *************************************
+* HC Compression
+***************************************/
+/* Update chains up to ip (excluded) */
+static void ZSTD_HC_insert (ZSTD_HC_CCtx* zc, const BYTE* ip)
+{
+ U16* chainTable = zc->chainTable;
+ U32* HashTable = zc->hashTable;
+ const BYTE* const base = zc->base;
+ const U32 target = (U32)(ip - base);
+ U32 idx = zc->nextToUpdate;
+
+ while(idx < target)
+ {
+ U32 h = ZSTD_HC_hashPtr(base+idx);
+ size_t delta = idx - HashTable[h];
+ if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
+ DELTANEXTU16(idx) = (U16)delta;
+ HashTable[h] = idx;
+ idx++;
+ }
+
+ zc->nextToUpdate = target;
+}
+
+
+static size_t ZSTD_HC_insertAndFindBestMatch (
+ ZSTD_HC_CCtx* zc, /* Index table will be updated */
+ const BYTE* ip, const BYTE* const iLimit,
+ const BYTE** matchpos,
+ const U32 maxNbAttempts)
+{
+ U16* const chainTable = zc->chainTable;
+ U32* const HashTable = zc->hashTable;
+ const BYTE* const base = zc->base;
+ const BYTE* const dictBase = zc->dictBase;
+ const U32 dictLimit = zc->dictLimit;
+ const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1);
+ U32 matchIndex;
+ const BYTE* match;
+ int nbAttempts=maxNbAttempts;
+ size_t ml=0;
+
+ /* HC4 match finder */
+ ZSTD_HC_insert(zc, ip);
+ matchIndex = HashTable[ZSTD_HC_hashPtr(ip)];
+
+ while ((matchIndex>=lowLimit) && (nbAttempts))
+ {
+ nbAttempts--;
+ if (matchIndex >= dictLimit)
+ {
+ match = base + matchIndex;
+ if (*(match+ml) == *(ip+ml)
+ && (MEM_read32(match) == MEM_read32(ip)))
+ {
+ const size_t mlt = ZSTD_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
+ if (mlt > ml) { ml = mlt; *matchpos = match; }
+ }
+ }
+ else
+ {
+ match = dictBase + matchIndex;
+ if (MEM_read32(match) == MEM_read32(ip))
+ {
+ size_t mlt;
+ const BYTE* vLimit = ip + (dictLimit - matchIndex);
+ if (vLimit > iLimit) vLimit = iLimit;
+ mlt = ZSTD_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
+ if ((ip+mlt == vLimit) && (vLimit < iLimit))
+ mlt += ZSTD_count(ip+mlt, base+dictLimit, iLimit);
+ if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
+ }
+ }
+ matchIndex -= DELTANEXTU16(matchIndex);
+ }
+
+ return ml;
+}
+
+
+size_t ZSTD_HC_InsertAndGetWiderMatch (
+ ZSTD_HC_CCtx* zc,
+ const BYTE* const ip,
+ const BYTE* const iLowLimit,
+ const BYTE* const iHighLimit,
+ size_t longest,
+ const BYTE** matchpos,
+ const BYTE** startpos,
+ const int maxNbAttempts)
+{
+ U16* const chainTable = zc->chainTable;
+ U32* const HashTable = zc->hashTable;
+ const BYTE* const base = zc->base;
+ const U32 dictLimit = zc->dictLimit;
+ const BYTE* const lowPrefixPtr = base + dictLimit;
+ const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1);
+ const BYTE* const dictBase = zc->dictBase;
+ U32 matchIndex;
+ int nbAttempts = maxNbAttempts;
+ int delta = (int)(ip-iLowLimit);
+
+
+ /* First Match */
+ ZSTD_HC_insert(zc, ip);
+ matchIndex = HashTable[ZSTD_HC_hashPtr(ip)];
+
+ while ((matchIndex>=lowLimit) && (nbAttempts))
+ {
+ nbAttempts--;
+ if (matchIndex >= dictLimit)
+ {
+ const BYTE* matchPtr = base + matchIndex;
+ if (*(iLowLimit + longest) == *(matchPtr - delta + longest))
+ if (MEM_read32(matchPtr) == MEM_read32(ip))
+ {
+ size_t mlt = MINMATCH + ZSTD_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+ int back = 0;
+
+ while ((ip+back>iLowLimit)
+ && (matchPtr+back > lowPrefixPtr)
+ && (ip[back-1] == matchPtr[back-1]))
+ back--;
+
+ mlt -= back;
+
+ if (mlt > longest)
+ {
+ longest = mlt;
+ *matchpos = matchPtr+back;
+ *startpos = ip+back;
+ }
+ }
+ }
+ else
+ {
+ const BYTE* matchPtr = dictBase + matchIndex;
+ if (MEM_read32(matchPtr) == MEM_read32(ip))
+ {
+ size_t mlt;
+ int back=0;
+ const BYTE* vLimit = ip + (dictLimit - matchIndex);
+ if (vLimit > iHighLimit) vLimit = iHighLimit;
+ mlt = ZSTD_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+ if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
+ mlt += ZSTD_count(ip+mlt, base+dictLimit, iHighLimit);
+ while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
+ mlt -= back;
+ if (mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
+ }
+ }
+ matchIndex -= DELTANEXTU16(matchIndex);
+ }
+
+ return longest;
+}
+
+
+static size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+ seqStore_t* seqStorePtr = &(ctx->seqStore);
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart + 1;
+ const BYTE* anchor = istart;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - 8;
+
+ size_t offset_2=4, offset_1=4;
+ const U32 maxSearches = 1 << ctx->compressionLevel;
+
+ /* init */
+ ZSTD_resetSeqStore(seqStorePtr);
+
+ /* Main Search Loop */
+ while (ip < ilimit)
+ {
+ const BYTE* match;
+ size_t matchLength = ZSTD_HC_insertAndFindBestMatch(ctx, ip, ilimit, &match, maxSearches);
+ if (!matchLength) { ip++; continue; }
+ /* save match */
+ {
+ size_t litLength = ip-anchor;
+ size_t offset = ip-match;
+ if (litLength) offset_2 = offset_1;
+ if (offset == offset_2) offset = 0;
+ offset_2 = offset_1;
+ offset_1 = ip-match;
+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH);
+ ip += matchLength;
+ anchor = ip;
+ }
+ }
+
+ /* Last Literals */
+ {
+ size_t lastLLSize = iend - anchor;
+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+ }
+
+ /* Finale compression stage */
+ return ZSTD_compressSequences((BYTE*)dst, maxDstSize,
+ seqStorePtr, srcSize);
+}
+
+static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
+ void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize)
+{
+ static const size_t blockSize = 128 KB;
+ size_t remaining = srcSize;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* op = ostart;
+ BYTE* const oend = op + maxDstSize;
+
+ while (remaining > blockSize)
+ {
+ size_t cSize = ZSTD_HC_compressBlock(ctxPtr, op+3, oend-op, ip, blockSize);
+
+ if (cSize == 0)
+ {
+ cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
+ }
+ else
+ {
+ op[0] = (BYTE)(cSize>>16);
+ op[1] = (BYTE)(cSize>>8);
+ op[2] = (BYTE)cSize;
+ op[0] += (BYTE)(bt_compressed << 6); /* is a compressed block */
+ cSize += 3;
+ }
+
+ remaining -= blockSize;
+ ip += blockSize;
+ op += cSize;
+ if (ZSTD_isError(cSize)) return cSize;
+ }
+
+ /* last block */
+ {
+ size_t cSize = ZSTD_HC_compressBlock(ctxPtr, op+3, oend-op, ip, remaining);
+
+ if (cSize == 0)
+ {
+ cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, remaining); /* block is not compressible */
+ }
+ else
+ {
+ op[0] = (BYTE)(cSize>>16);
+ op[1] = (BYTE)(cSize>>8);
+ op[2] = (BYTE)cSize;
+ op[0] += (BYTE)(bt_compressed << 6); /* is a compressed block */
+ cSize += 3;
+ }
+
+ op += cSize;
+ if (ZSTD_isError(cSize)) return cSize;
+ }
+
+ return op-ostart;
+}
+
+
+size_t ZSTD_HC_loadDict(ZSTD_HC_CCtx* ctx, const void* dictionary, size_t dictSize)
+{
+ /* TBD */
+ (void)ctx; (void)dictionary; (void)dictSize;
+ return 0;
+}
+
+static void ZSTD_HC_setExternalDict(ZSTD_HC_CCtx* ctxPtr, const void* newBlock)
+{
+ if (ctxPtr->end >= ctxPtr->base + 4)
+ ZSTD_HC_insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
+ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
+ ctxPtr->lowLimit = ctxPtr->dictLimit;
+ ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
+ ctxPtr->dictBase = ctxPtr->base;
+ ctxPtr->base = newBlock - ctxPtr->dictLimit;
+ ctxPtr->end = newBlock;
+ ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
+}
+
+size_t ZSTD_HC_compress_continue (ZSTD_HC_CCtx* ctxPtr,
+ void* dst, size_t dstSize,
+ const void* src, size_t srcSize)
+{
+ /* Check overflow */
+ if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
+ {
+ size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
+ if (dictSize > 64 KB) dictSize = 64 KB;
+
+ ZSTD_HC_loadDict(ctxPtr, ctxPtr->end - dictSize, dictSize);
+ }
+
+ /* Check if blocks follow each other */
+ if ((const BYTE*)src != ctxPtr->end)
+ ZSTD_HC_setExternalDict(ctxPtr, (const BYTE*)src);
+
+ /* Check overlapping src/dictionary space (typical of cycling buffers) */
+ {
+ const BYTE* sourceEnd = (const BYTE*) src + srcSize;
+ const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
+ const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
+ if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd))
+ {
+ if (sourceEnd > dictEnd) sourceEnd = dictEnd;
+ ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
+ if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
+ }
+ }
+
+ return ZSTD_HC_compress_generic (ctxPtr, dst, dstSize, src, srcSize);
+}
+
+
+size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, unsigned compressionLevel, const void* src)
+{
+ /* Sanity check */
+ if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
+
+ /* Init */
+ ZSTD_HC_resetCCtx(ctx, compressionLevel, src);
+
+ /* Write Header */
+ MEM_writeLE32(dst, ZSTD_magicNumber);
+
+ return 4;
+}
+
+size_t ZSTD_HC_compressCCtx (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* op = ostart;
+
+ /* Header */
+ size_t oSize = ZSTD_HC_compressBegin(ctx, dst, maxDstSize, compressionLevel, src);
+ if(ZSTD_isError(oSize)) return oSize;
+ op += oSize;
+ maxDstSize -= oSize;
+
+ /* body (compression) */
+ op += ZSTD_HC_compress_generic (ctx, op, maxDstSize, src, srcSize);
+ if(ZSTD_isError(oSize)) return oSize;
+ op += oSize;
+ maxDstSize -= oSize;
+
+ /* Close frame */
+ oSize = ZSTD_compressEnd((ZSTD_CCtx*)ctx, op, maxDstSize);
+ if(ZSTD_isError(oSize)) return oSize;
+ op += oSize;
+
+ return (op - ostart);
+}
+
+size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel)
+{
+ ZSTD_HC_CCtx ctxBody;
+ return ZSTD_HC_compressCCtx(&ctxBody, dst, maxDstSize, src, srcSize, compressionLevel);
+}
+
+
+
+/**************************************
+* Streaming Functions
+**************************************/
+
+
+
+/* dictionary saving */
+
+size_t ZSTD_HC_saveDict (ZSTD_HC_CCtx* ctx, void* safeBuffer, size_t dictSize)
+{
+ /* TBD */
+ (void)ctx; (void)safeBuffer; (void)dictSize;
+ return 0;
+}
+
--- /dev/null
+/*
+ zstdhc - high compression variant
+ Header File
+ Copyright (C) 2015, Yann Collet.
+
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - zstd source repository : http://www.zstd.net
+*/
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* *************************************
+* Includes
+***************************************/
+#include <stddef.h> /* size_t */
+
+
+/* *************************************
+* Simple function
+***************************************/
+/**
+ZSTD_HC_compress() :
+ Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'.
+ Destination buffer must be already allocated.
+ Compression runs faster if maxDstSize >= ZSTD_compressBound(srcSize).
+ @return : the number of bytes written into buffer 'dst'
+ or an error code if it fails (which can be tested using ZSTD_isError())
+*/
+size_t ZSTD_HC_compress(void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize,
+ unsigned compressionLevel);
+
+
+/* *************************************
+* Advanced functions
+***************************************/
+typedef struct ZSTD_HC_CCtx_s ZSTD_HC_CCtx; /* incomplete type */
+ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void);
+size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx);
+
+/**
+ZSTD_HC_compressCCtx() :
+ Same as ZSTD_compress(), but requires a ZSTD_HC_CCtx working space already allocated
+*/
+size_t ZSTD_HC_compressCCtx(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel);
+
+
+#if defined (__cplusplus)
+}
+#endif
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
-VERSION?= v0.2.0
+VERSION?= 0.3.0
DESTDIR?=
PREFIX ?= /usr/local
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen
-zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
+zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
-/***************************************
+/* **************************************
* Compiler Options
-***************************************/
+****************************************/
/* Disable some Visual warning messages */
#define _CRT_SECURE_NO_WARNINGS /* fopen */
-// Unix Large Files support (>4GB)
+/* Unix Large Files support (>4GB) */
#define _FILE_OFFSET_BITS 64
-#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
+#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
# define _LARGEFILE_SOURCE
-#elif ! defined(__LP64__) // No point defining Large file for 64 bit
+#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
# define _LARGEFILE64_SOURCE
#endif
-// S_ISREG & gettimeofday() are not supported by MSVC
+/* S_ISREG & gettimeofday() are not supported by MSVC */
#if defined(_MSC_VER) || defined(_WIN32)
# define BMK_LEGACY_TIMER 1
#endif
-/**************************************
+/* *************************************
* Includes
-**************************************/
+***************************************/
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> // fprintf, fopen, ftello64
# include <sys/time.h> // gettimeofday
#endif
+#include "mem.h"
#include "zstd.h"
+#include "zstdhc.h"
#include "xxhash.h"
-/**************************************
+/* *************************************
* Compiler specifics
-**************************************/
+***************************************/
#if !defined(S_ISREG)
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
-/**************************************
-* Basic Types
-**************************************/
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
-# include <stdint.h>
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
-#else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
- typedef unsigned long long U64;
-#endif
-
-
-/**************************************
+/* *************************************
* Constants
-**************************************/
+***************************************/
#define NBLOOPS 3
#define TIMELOOP 2500
static U32 prime2 = 2246822519U;
-/**************************************
+/* *************************************
* Macros
-**************************************/
+***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-/**************************************
+/* *************************************
* Benchmark Parameters
-**************************************/
+***************************************/
static int nbIterations = NBLOOPS;
static size_t g_blockSize = 0;
}
-/*********************************************************
+/* ********************************************************
* Private functions
-*********************************************************/
+**********************************************************/
#if defined(BMK_LEGACY_TIMER)
}
-
-/*********************************************************
+/* ********************************************************
* Data generator
-*********************************************************/
+**********************************************************/
/* will hopefully be converted into ROL instruction by compiler */
static U32 BMK_rotl32(unsigned val32, unsigned nbBits) { return((val32 << nbBits) | (val32 >> (32 - nbBits))); }
}
-/*********************************************************
+/* ********************************************************
* Bench functions
-*********************************************************/
+**********************************************************/
typedef struct
{
char* srcPtr;
} blockParam_t;
+typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel);
+
+static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel)
+{
+ (void)compressionLevel;
+ return ZSTD_compress(dst, maxDstSize, src, srcSize);
+}
+
#define MIN(a,b) (a<b ? a : b)
static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
+ compressor_t compressor;
U64 crcOrig;
/* Init */
- (void)cLevel;
+ if (cLevel <= 1) compressor = local_compress_fast;
+ else compressor = ZSTD_HC_compress;
/* Memory allocation & restrictions */
if (!compressedBuffer || !resultBuffer || !blockTable)
while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (blockNb=0; blockNb<nbBlocks; blockNb++)
- blockTable[blockNb].cSize = ZSTD_compress(blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
- blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
+ blockTable[blockNb].cSize = compressor(blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel);
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck)
{
- unsigned i;
+ unsigned u;
+ unsigned eBlockSize = MIN(65536*2, blockSize);
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
- for (i=0; i<srcSize; i++)
+ for (u=0; u<srcSize; u++)
{
- if (((BYTE*)srcBuffer)[i] != ((BYTE*)resultBuffer)[i])
+ if (((BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u])
{
- printf("\nDecoding error at pos %u \n", i);
+ printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
}
}
#define CACHELINE 64
-/**************************************
-* Complex types
-**************************************/
-typedef enum { bt_compressed, bt_raw, bt_rle, bt_crc } bType_t;
-
-
/**************************************
* Macros
**************************************/
}
-unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename)
+unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, unsigned cLevel)
{
U64 filesize = 0;
U64 compressedfilesize = 0;
/* Init */
+ (void)cLevel;
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
/* Allocate Memory */
#endif
-/**************************************
+/* *************************************
* Special i/o constants
**************************************/
#define nullString "null"
#endif
-/**************************************
+/* *************************************
* Parameters
-**************************************/
+***************************************/
void FIO_overwriteMode(void);
void FIO_setNotificationLevel(unsigned level);
-/**************************************
+/* *************************************
* Stream/File functions
-**************************************/
-unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename);
+***************************************/
+unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, unsigned cLevel);
unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename);
-/*
+/**
FIO_compressFilename :
- result : size of compressed file
+ @result : size of compressed file
FIO_decompressFilename :
- result : size of regenerated file
+ @result : size of regenerated file
*/
#if defined (__cplusplus)
}
-#endif
\ No newline at end of file
+#endif
main_pause=0;
unsigned fileNameStart = 0;
unsigned nbFiles = 0;
+ unsigned cLevel = 0;
const char* programName = argv[0];
const char* inFileName = NULL;
const char* outFileName = NULL;
while (argument[0]!=0)
{
+ /* compression Level */
+ if ((*argument>='0') && (*argument<='9'))
+ {
+ cLevel = 0;
+ while ((*argument >= '0') && (*argument <= '9'))
+ {
+ cLevel *= 10;
+ cLevel += *argument - '0';
+ argument++;
+ }
+ continue;
+ }
+
switch(argument[0])
{
/* Display help */
if (!strcmp(inFileName, stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
/* Check if benchmark is selected */
- if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, 0); goto _end; }
+ if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, cLevel); goto _end; }
/* No output filename ==> try to select one automatically (when possible) */
while (!outFileName)
if (decode)
FIO_decompressFilename(outFileName, inFileName);
else
- FIO_compressFilename(outFileName, inFileName);
+ FIO_compressFilename(outFileName, inFileName, cLevel);
_end:
if (main_pause) waitEnter();
projects / thirdparty / zstd.git / commitdiff
