uncoupled maxdlog and chainlog

diff --git a/lib/zstdhc.c b/lib/zstdhc.c
index a635e1afc62f347ca76261de5bcd12cf72975ed5..73a22f5cc8e357f262563333ca5a60a77e2f8b69 100644 (file)
--- a/lib/zstdhc.c
+++ b/lib/zstdhc.c
@@ -52,17 +52,18 @@ static const U32 ZSTD_HC_compressionLevel_default = 9;
 *  Local Constants
 ***************************************/
 #define MINMATCH 4
-#define MAX_DISTANCE 65535   /* <=== To be changed (dynamic ?) */
+#define MAXD_LOG 19
+#define MAX_DISTANCE (1 << MAXD_LOG)   /* <=== dynamic ? */
 
-#define DICTIONARY_LOGSIZE 16
-#define MAXD (1<<DICTIONARY_LOGSIZE)
-#define MAXD_MASK (MAXD - 1)
+#define CHAIN_LOG 16
+#define CHAIN_SIZE (1<<CHAIN_LOG)
+#define CHAIN_MASK (CHAIN_SIZE - 1)
 
-#define HASH_LOG (DICTIONARY_LOGSIZE-1)
+#define HASH_LOG (CHAIN_LOG-1)
 #define HASHTABLESIZE (1 << HASH_LOG)
 #define HASH_MASK (HASHTABLESIZE - 1)
 
-static const U32 g_maxCompressionLevel = 19;
+static const U32 g_maxCompressionLevel = MAXD_LOG;
 
 #define KB *1024
 #define MB *1024*1024
@@ -77,16 +78,15 @@ static const U32 g_maxCompressionLevel = 19;
 struct ZSTD_HC_CCtx_s
 {
     U32   hashTable[HASHTABLESIZE];
-    U16   chainTable[MAXD];
+    U16   chainTable[CHAIN_SIZE];
     const BYTE* end;        /* next block here to continue on current prefix */
-    const BYTE* base;       /* All index relative to this position */
-    const BYTE* dictBase;   /* alternate base for extDict */
-    BYTE* inputBuffer;      /* deprecated */
+    const BYTE* base;       /* All regular indexes relative to this position */
+    const BYTE* dictBase;   /* extDict indexes relative to this position */
     U32   dictLimit;        /* below that point, need extDict */
-    U32   lowLimit;         /* below that point, no more dict */
+    U32   lowLimit;         /* below that point, no more data */
     U32   nextToUpdate;     /* index from which to continue dictionary update */
     U32   compressionLevel;
-    seqStore_t seqStore;
+    seqStore_t seqStore;    /* sequences storage ptrs */
        BYTE buffer[WORKPLACESIZE];
 };
 
@@ -98,18 +98,18 @@ ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void)
 
 size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx) { free(cctx); return 0; }
 
-static void ZSTD_HC_resetCCtx (ZSTD_HC_CCtx* zc, U32 compressionLevel, const BYTE* start)
+static void ZSTD_HC_resetCCtx (ZSTD_HC_CCtx* zc, U32 compressionLevel, const void* start)
 {
     if (compressionLevel==0) compressionLevel = ZSTD_HC_compressionLevel_default;
     if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel;
     memset(zc->hashTable, 0, sizeof(zc->hashTable));
     memset(zc->chainTable, 0xFF, sizeof(zc->chainTable));
-    zc->nextToUpdate = 64 KB;
-    zc->base = start - 64 KB;
-    zc->end = start;
-    zc->dictBase = start - 64 KB;
-    zc->dictLimit = 64 KB;
-    zc->lowLimit = 64 KB;
+    zc->nextToUpdate = MAX_DISTANCE;
+    zc->end = (const BYTE*)start;
+    zc->base = zc->end - MAX_DISTANCE;
+    zc->dictBase = zc->base;
+    zc->dictLimit = MAX_DISTANCE;
+    zc->lowLimit = MAX_DISTANCE;
     zc->compressionLevel = compressionLevel;
        zc->seqStore.buffer = zc->buffer;
     zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
@@ -124,8 +124,8 @@ static void ZSTD_HC_resetCCtx (ZSTD_HC_CCtx* zc, U32 compressionLevel, const BYT
 *  Local Macros
 ***************************************/
 #define HASH_FUNCTION(u)       (((u) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
-//#define DELTANEXTU16(p)        chainTable[(p) & MAXD_MASK]   /* flexible, MAXD dependent */
-#define DELTANEXTU16(p)        chainTable[(U16)(p)]   /* faster */
+//#define DELTANEXTU16(d)        chainTable[(d) & MAXD_MASK]   /* flexible, CHAINSIZE dependent */
+#define DELTANEXTU16(d)        chainTable[(U16)(d)]   /* faster, specific to CHAINLOG==16 */
 
 static U32 ZSTD_HC_hashPtr(const void* ptr) { return HASH_FUNCTION(MEM_read32(ptr)); }
 
@@ -167,7 +167,7 @@ static size_t ZSTD_HC_insertAndFindBestMatch (
     const BYTE* const base = zc->base;
     const BYTE* const dictBase = zc->dictBase;
     const U32 dictLimit = zc->dictLimit;
-    const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1);
+    const U32 lowLimit = (zc->lowLimit + MAX_DISTANCE > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (MAX_DISTANCE - 1);
     U32 matchIndex;
     const BYTE* match;
     int nbAttempts=maxNbAttempts;
@@ -204,6 +204,7 @@ static size_t ZSTD_HC_insertAndFindBestMatch (
                 if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; }   /* virtual matchpos */
             }
         }
+        if (base + matchIndex <= ip - CHAIN_SIZE) break;
         matchIndex -= DELTANEXTU16(matchIndex);
     }
 
@@ -226,7 +227,7 @@ size_t ZSTD_HC_InsertAndGetWiderMatch (
     const BYTE* const base = zc->base;
     const U32 dictLimit = zc->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
-    const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1);
+    const U32 lowLimit = (zc->lowLimit + MAX_DISTANCE > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (MAX_DISTANCE - 1);
     const BYTE* const dictBase = zc->dictBase;
     U32   matchIndex;
     int nbAttempts = maxNbAttempts;
@@ -281,6 +282,8 @@ size_t ZSTD_HC_InsertAndGetWiderMatch (
                 if (mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
             }
         }
+        if (base + matchIndex <= ip - CHAIN_SIZE)
+            matchIndex -= MAX_DISTANCE;  /* ensures it gets eliminated on next test */
         matchIndex -= DELTANEXTU16(matchIndex);
     }
 
@@ -409,8 +412,8 @@ static void ZSTD_HC_setExternalDict(ZSTD_HC_CCtx* ctxPtr, const void* newBlock)
     ctxPtr->lowLimit  = ctxPtr->dictLimit;
     ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
     ctxPtr->dictBase  = ctxPtr->base;
-    ctxPtr->base = newBlock - ctxPtr->dictLimit;
-    ctxPtr->end  = newBlock;
+    ctxPtr->base = (const BYTE*)newBlock - ctxPtr->dictLimit;
+    ctxPtr->end  = (const BYTE*)newBlock;
     ctxPtr->nextToUpdate = ctxPtr->dictLimit;   /* match referencing will resume from there */
 }
 
@@ -422,7 +425,7 @@ size_t ZSTD_HC_compress_continue (ZSTD_HC_CCtx* ctxPtr,
     if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
     {
         size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
-        if (dictSize > 64 KB) dictSize = 64 KB;
+        if (dictSize > MAX_DISTANCE) dictSize = MAX_DISTANCE;
 
         ZSTD_HC_loadDict(ctxPtr, ctxPtr->end - dictSize, dictSize);
     }
@@ -498,9 +501,6 @@ size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t sr
 /**************************************
 *  Streaming Functions
 **************************************/
-
-
-
 /* dictionary saving */
 
 size_t ZSTD_HC_saveDict (ZSTD_HC_CCtx* ctx, void* safeBuffer, size_t dictSize)

diff --git a/programs/Makefile b/programs/Makefile
index 7295425bc506fe15ca9cc576e8727a052a781c18..891c55f75a0bb769f65c788ebf552aa6cb293eeb 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -61,7 +61,7 @@ all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen
 zstd  : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
        $(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
+zstd32: $(ZSTDDIR)/zstd.c  $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
        $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
 
 fullbench  : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c fullbench.c

diff --git a/programs/bench.c b/programs/bench.c
index 86a536851d0f147aa986cfd728032f87040ea302..aed0a446d644f9d3402e2c6c59aecbbf7b7d6389 100644 (file)
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -48,15 +48,15 @@
 ***************************************/
 #include <stdlib.h>      /* malloc, free */
 #include <string.h>      /* memset */
-#include <stdio.h>       // fprintf, fopen, ftello64
-#include <sys/types.h>   // stat64
-#include <sys/stat.h>    // stat64
+#include <stdio.h>       /* fprintf, fopen, ftello64 */
+#include <sys/types.h>   /* stat64 */
+#include <sys/stat.h>    /* stat64 */
 
-// Use ftime() if gettimeofday() is not available on your target
+/* Use ftime() if gettimeofday() is not available */
 #if defined(BMK_LEGACY_TIMER)
-#  include <sys/timeb.h>   // timeb, ftime
+#  include <sys/timeb.h>   /* timeb, ftime */
 #else
-#  include <sys/time.h>    // gettimeofday
+#  include <sys/time.h>    /* gettimeofday */
 #endif
 
 #include "mem.h"
@@ -249,13 +249,9 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
     const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
     void* const compressedBuffer = malloc(maxCompressedSize);
     void* const resultBuffer = malloc(srcSize);
-    compressor_t compressor;
+    const compressor_t compressor = (cLevel <= 1) ? local_compress_fast : ZSTD_HC_compress;
     U64 crcOrig;
 
-    /* Init */
-    if (cLevel <= 1) compressor = local_compress_fast;
-    else compressor = ZSTD_HC_compress;
-
     /* Memory allocation & restrictions */
     if (!compressedBuffer || !resultBuffer || !blockTable)
     {
@@ -356,7 +352,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
             if (crcOrig!=crcCheck)
             {
                 unsigned u;
-                unsigned eBlockSize = MIN(65536*2, blockSize);
+                unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
                 DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
                 for (u=0; u<srcSize; u++)
                 {
@@ -465,10 +461,10 @@ static int BMK_benchOneFile(char* inFileName, int cLevel)
         return 13;
     }
 
-    // Bench
+    /* Bench */
     result = BMK_benchMem(srcBuffer, benchedSize, inFileName, cLevel);
 
-    // End
+    /* clean up */
     free(srcBuffer);
     DISPLAY("\n");
     return result;

diff --git a/programs/fileio.c b/programs/fileio.c
index 710c89719f73462b29c4f1cf70b9fa6739cf80b2..2c768253ed60d6f9b016c5e7de46f2378a7e364d 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -29,9 +29,9 @@
   The license of this file is GPLv2.
 */
 
-/**************************************
+/* *************************************
 *  Tuning options
-**************************************/
+***************************************/
 #ifndef ZSTD_LEGACY_SUPPORT
 /**LEGACY_SUPPORT :
 *  decompressor can decode older formats (starting from Zstd 0.1+) */
@@ -39,9 +39,9 @@
 #endif // ZSTD_LEGACY_SUPPORT
 
 
-/**************************************
+/* *************************************
 *  Compiler Options
-**************************************/
+***************************************/
 /* Disable some Visual warning messages */
 #ifdef _MSC_VER
 #  define _CRT_SECURE_NO_WARNINGS
@@ -55,9 +55,9 @@
 #define _POSIX_SOURCE 1        /* enable fileno() within <stdio.h> on unix */
 
 
-/**************************************
+/* *************************************
 *  Includes
-**************************************/
+***************************************/
 #include <stdio.h>     /* fprintf, fopen, fread, _fileno, stdin, stdout */
 #include <stdlib.h>    /* malloc, free */
 #include <string.h>    /* strcmp, strlen */
@@ -69,12 +69,12 @@
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
 #  include "zstd_v01.h"  /* legacy */
-#endif // ZSTD_LEGACY_SUPPORT
+#endif
 
 
-/**************************************
+/* *************************************
 *  OS-specific Includes
-**************************************/
+***************************************/
 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
 #  include <fcntl.h>    /* _O_BINARY */
 #  include <io.h>       /* _setmode, _isatty */
@@ -90,9 +90,9 @@
 #endif
 
 
-/**************************************
+/* *************************************
 *  Constants
-**************************************/
+***************************************/
 #define KB *(1U<<10)
 #define MB *(1U<<20)
 #define GB *(1U<<30)
@@ -116,9 +116,9 @@ static const unsigned FIO_blockHeaderSize = 3;
 #define CACHELINE 64
 
 
-/**************************************
+/* *************************************
 *  Macros
-**************************************/
+***************************************/
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static U32 g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result + interaction + warnings;   3 : + progression;   4 : + information */
@@ -131,19 +131,21 @@ static const unsigned refreshRate = 150;
 static clock_t g_time = 0;
 
 
-/**************************************
+/* *************************************
 *  Local Parameters
-**************************************/
+***************************************/
 static U32 g_overwrite = 0;
 
 void FIO_overwriteMode(void) { g_overwrite=1; }
 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 
 
-/**************************************
+/* *************************************
 *  Exceptions
-**************************************/
-#define DEBUG 0
+***************************************/
+#ifndef DEBUG
+#  define DEBUG 0
+#endif
 #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
 #define EXM_THROW(error, ...)                                             \
 {                                                                         \
@@ -155,9 +157,9 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 }
 
 
-/**************************************
+/* *************************************
 *  Functions
-**************************************/
+***************************************/
 static unsigned FIO_GetMilliSpan(clock_t nPrevious)
 {
     clock_t nCurrent = clock();