From: Yann Collet Date: Thu, 22 Oct 2015 15:55:40 +0000 (+0100) Subject: uncoupled maxdlog and chainlog X-Git-Tag: zstd-0.3.0^2~36 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=eeb8ba18d17ad08f28705a45777f002dd9bd518a;p=thirdparty%2Fzstd.git uncoupled maxdlog and chainlog --- diff --git a/lib/zstdhc.c b/lib/zstdhc.c index a635e1afc..73a22f5cc 100644 --- a/lib/zstdhc.c +++ b/lib/zstdhc.c @@ -52,17 +52,18 @@ static const U32 ZSTD_HC_compressionLevel_default = 9; * Local Constants ***************************************/ #define MINMATCH 4 -#define MAX_DISTANCE 65535 /* <=== To be changed (dynamic ?) */ +#define MAXD_LOG 19 +#define MAX_DISTANCE (1 << MAXD_LOG) /* <=== dynamic ? */ -#define DICTIONARY_LOGSIZE 16 -#define MAXD (1< g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel; memset(zc->hashTable, 0, sizeof(zc->hashTable)); memset(zc->chainTable, 0xFF, sizeof(zc->chainTable)); - zc->nextToUpdate = 64 KB; - zc->base = start - 64 KB; - zc->end = start; - zc->dictBase = start - 64 KB; - zc->dictLimit = 64 KB; - zc->lowLimit = 64 KB; + zc->nextToUpdate = MAX_DISTANCE; + zc->end = (const BYTE*)start; + zc->base = zc->end - MAX_DISTANCE; + zc->dictBase = zc->base; + zc->dictLimit = MAX_DISTANCE; + zc->lowLimit = MAX_DISTANCE; zc->compressionLevel = compressionLevel; zc->seqStore.buffer = zc->buffer; zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer); @@ -124,8 +124,8 @@ static void ZSTD_HC_resetCCtx (ZSTD_HC_CCtx* zc, U32 compressionLevel, const BYT * Local Macros ***************************************/ #define HASH_FUNCTION(u) (((u) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) -//#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */ -#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ +//#define DELTANEXTU16(d) chainTable[(d) & MAXD_MASK] /* flexible, CHAINSIZE dependent */ +#define DELTANEXTU16(d) chainTable[(U16)(d)] /* faster, specific to CHAINLOG==16 */ static U32 ZSTD_HC_hashPtr(const void* ptr) { return HASH_FUNCTION(MEM_read32(ptr)); } @@ -167,7 +167,7 @@ static size_t ZSTD_HC_insertAndFindBestMatch ( const BYTE* const base = zc->base; const BYTE* const dictBase = zc->dictBase; const U32 dictLimit = zc->dictLimit; - const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1); + const U32 lowLimit = (zc->lowLimit + MAX_DISTANCE > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (MAX_DISTANCE - 1); U32 matchIndex; const BYTE* match; int nbAttempts=maxNbAttempts; @@ -204,6 +204,7 @@ static size_t ZSTD_HC_insertAndFindBestMatch ( if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ } } + if (base + matchIndex <= ip - CHAIN_SIZE) break; matchIndex -= DELTANEXTU16(matchIndex); } @@ -226,7 +227,7 @@ size_t ZSTD_HC_InsertAndGetWiderMatch ( const BYTE* const base = zc->base; const U32 dictLimit = zc->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1); + const U32 lowLimit = (zc->lowLimit + MAX_DISTANCE > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (MAX_DISTANCE - 1); const BYTE* const dictBase = zc->dictBase; U32 matchIndex; int nbAttempts = maxNbAttempts; @@ -281,6 +282,8 @@ size_t ZSTD_HC_InsertAndGetWiderMatch ( if (mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } } } + if (base + matchIndex <= ip - CHAIN_SIZE) + matchIndex -= MAX_DISTANCE; /* ensures it gets eliminated on next test */ matchIndex -= DELTANEXTU16(matchIndex); } @@ -409,8 +412,8 @@ static void ZSTD_HC_setExternalDict(ZSTD_HC_CCtx* ctxPtr, const void* newBlock) ctxPtr->lowLimit = ctxPtr->dictLimit; ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; + ctxPtr->base = (const BYTE*)newBlock - ctxPtr->dictLimit; + ctxPtr->end = (const BYTE*)newBlock; ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ } @@ -422,7 +425,7 @@ size_t ZSTD_HC_compress_continue (ZSTD_HC_CCtx* ctxPtr, if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; - if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize > MAX_DISTANCE) dictSize = MAX_DISTANCE; ZSTD_HC_loadDict(ctxPtr, ctxPtr->end - dictSize, dictSize); } @@ -498,9 +501,6 @@ size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t sr /************************************** * Streaming Functions **************************************/ - - - /* dictionary saving */ size_t ZSTD_HC_saveDict (ZSTD_HC_CCtx* ctx, void* safeBuffer, size_t dictSize) diff --git a/programs/Makefile b/programs/Makefile index 7295425bc..891c55f75 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -61,7 +61,7 @@ all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c $(CC) $(FLAGS) $^ -o $@$(EXT) -zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c +zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) fullbench : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c fullbench.c diff --git a/programs/bench.c b/programs/bench.c index 86a536851..aed0a446d 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -48,15 +48,15 @@ ***************************************/ #include /* malloc, free */ #include /* memset */ -#include // fprintf, fopen, ftello64 -#include // stat64 -#include // stat64 +#include /* fprintf, fopen, ftello64 */ +#include /* stat64 */ +#include /* stat64 */ -// Use ftime() if gettimeofday() is not available on your target +/* Use ftime() if gettimeofday() is not available */ #if defined(BMK_LEGACY_TIMER) -# include // timeb, ftime +# include /* timeb, ftime */ #else -# include // gettimeofday +# include /* gettimeofday */ #endif #include "mem.h" @@ -249,13 +249,9 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize); void* const compressedBuffer = malloc(maxCompressedSize); void* const resultBuffer = malloc(srcSize); - compressor_t compressor; + const compressor_t compressor = (cLevel <= 1) ? local_compress_fast : ZSTD_HC_compress; U64 crcOrig; - /* Init */ - if (cLevel <= 1) compressor = local_compress_fast; - else compressor = ZSTD_HC_compress; - /* Memory allocation & restrictions */ if (!compressedBuffer || !resultBuffer || !blockTable) { @@ -356,7 +352,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i if (crcOrig!=crcCheck) { unsigned u; - unsigned eBlockSize = MIN(65536*2, blockSize); + unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize)); DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck); for (u=0; u on unix */ -/************************************** +/* ************************************* * Includes -**************************************/ +***************************************/ #include /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ @@ -69,12 +69,12 @@ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) # include "zstd_v01.h" /* legacy */ -#endif // ZSTD_LEGACY_SUPPORT +#endif -/************************************** +/* ************************************* * OS-specific Includes -**************************************/ +***************************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _O_BINARY */ # include /* _setmode, _isatty */ @@ -90,9 +90,9 @@ #endif -/************************************** +/* ************************************* * Constants -**************************************/ +***************************************/ #define KB *(1U<<10) #define MB *(1U<<20) #define GB *(1U<<30) @@ -116,9 +116,9 @@ static const unsigned FIO_blockHeaderSize = 3; #define CACHELINE 64 -/************************************** +/* ************************************* * Macros -**************************************/ +***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ @@ -131,19 +131,21 @@ static const unsigned refreshRate = 150; static clock_t g_time = 0; -/************************************** +/* ************************************* * Local Parameters -**************************************/ +***************************************/ static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } -/************************************** +/* ************************************* * Exceptions -**************************************/ -#define DEBUG 0 +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); #define EXM_THROW(error, ...) \ { \ @@ -155,9 +157,9 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } } -/************************************** +/* ************************************* * Functions -**************************************/ +***************************************/ static unsigned FIO_GetMilliSpan(clock_t nPrevious) { clock_t nCurrent = clock();