minor bt improvements

diff --git a/Makefile b/Makefile
index 7c3fab5ab9e492f26ce4cf935a5de49a573a37cd..2003da90cf247f64d3704c3328f4cf147e9bdaf6 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -32,7 +32,7 @@
 # ################################################################
 
 # Version number
-export VERSION := 0.3.4
+export VERSION := 0.3.5
 
 PRGDIR  = programs
 ZSTDDIR = lib

diff --git a/NEWS b/NEWS
index 1c7facdcc1a06e5a910e84ddcf964470743e3629..a5e18c12dcbafa077a5144ed360fbed4975e250f 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,6 @@
+v0.3.5
+minor generic compression improvements
+
 v0.3.4
 Faster fast cLevels
 

diff --git a/lib/zstd.h b/lib/zstd.h
index 115046df872bea7e37f61898ee2a40bc5d6ed6d9..b13400ecd4193b68a6f2b60720b44ef0dbd1dc7c 100644 (file)
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -48,7 +48,7 @@ extern "C" {
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
 #define ZSTD_VERSION_MINOR    3    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  4    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE  5    /* for tweaks, bug-fixes, or development */
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 unsigned ZSTD_versionNumber (void);
 

diff --git a/lib/zstdhc.c b/lib/zstdhc.c
index 44819e6f69f84b9f52731fee615e3ceefadc4d60..c1e7dcf6ee83636a4f3d51e19178b12c470b216f 100644 (file)
--- a/lib/zstdhc.c
+++ b/lib/zstdhc.c
@@ -161,7 +161,7 @@ static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
         zc->seqStore.buffer = (void*) (zc->contentTable + ((size_t)1 << contentLog));
     }
 
-    zc->nextToUpdate = 0;
+    zc->nextToUpdate = 1;
     zc->end = NULL;
     zc->base = NULL;
     zc->dictBase = NULL;
@@ -316,7 +316,7 @@ size_t ZSTD_HC_compressBlock_fast(ZSTD_HC_CCtx* ctx,
 ***************************************/
 /** ZSTD_HC_insertBt1 : add one ptr to tree
     @ip : assumed <= iend-8 */
-static void ZSTD_HC_insertBt1(ZSTD_HC_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares)
+static U32 ZSTD_HC_insertBt1(ZSTD_HC_CCtx* zc, const BYTE* ip, const U32 mls, const BYTE* const iend, U32 nbCompares)
 {
     U32* const hashTable = zc->hashTable;
     const U32 hashLog = zc->params.hashLog;
@@ -327,48 +327,61 @@ static void ZSTD_HC_insertBt1(ZSTD_HC_CCtx* zc, const BYTE* const ip, const U32
     U32 matchIndex  = hashTable[h];
     size_t commonLengthSmaller=0, commonLengthLarger=0;
     const BYTE* const base = zc->base;
-    const U32 current = (U32)(ip-base);
+    const BYTE* match = base + matchIndex;
+    U32 current = (U32)(ip-base);
     const U32 btLow = btMask >= current ? 0 : current - btMask;
     U32* smallerPtr = bt + 2*(current&btMask);
     U32* largerPtr  = bt + 2*(current&btMask) + 1;
     U32 dummy32;   /* to be nullified at the end */
     const U32 windowSize = 1 << zc->params.windowLog;
     const U32 windowLow = windowSize >= current ? 0 : current - windowSize;
+    U32 skip = 0;
 
-    hashTable[h] = (U32)(ip-base);   /* Update Hash Table */
+    if ( (current-matchIndex == 1)   /* RLE */
+        && ZSTD_read_ARCH(match) == ZSTD_read_ARCH(ip))
+    {
+        size_t cyclicLength = ZSTD_count(ip+sizeof(size_t), match+sizeof(size_t), iend) + sizeof(size_t);
+        skip = (U32)(cyclicLength - mls);    /* > 1 */
+        ip += skip;   /* last of segment */
+        smallerPtr += 2*skip;
+        largerPtr += 2*skip;
+    }
+
+    hashTable[h] = (U32)(ip - base);   /* Update Hash Table */
 
     while (nbCompares-- && (matchIndex > windowLow))
     {
         U32* nextPtr = bt + 2*(matchIndex & btMask);
-        const BYTE* match = base + matchIndex;
         size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger);   /* guaranteed minimum nb of common bytes */
 
+        match = base + matchIndex;
         matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
 
         if (ip+matchLength == iend)   /* equal : no way to know if inf or sup */
-            break;   /* just drop, to guarantee consistency (miss a little bit of compression) */
+            break;   /* just drop , to guarantee consistency (miss a bit of compression; if someone knows better, please tell) */
 
         if (match[matchLength] < ip[matchLength])
         {
             /* match is smaller than current */
             *smallerPtr = matchIndex;             /* update smaller idx */
             commonLengthSmaller = matchLength;    /* all smaller will now have at least this guaranteed common length */
+            if (matchIndex <= btLow) { smallerPtr=&dummy32; break; }   /* beyond tree size, stop the search */
             smallerPtr = nextPtr+1;               /* new "smaller" => larger of match */
-            if (matchIndex <= btLow) smallerPtr=&dummy32;  /* beyond tree size, stop the search */
-            matchIndex = (matchIndex <= btLow) ? windowLow : nextPtr[1];
+            matchIndex = nextPtr[1];              /* new matchIndex larger than previous (closer to current) */
         }
         else
         {
             /* match is larger than current */
             *largerPtr = matchIndex;
             commonLengthLarger = matchLength;
+            if (matchIndex <= btLow) { largerPtr=&dummy32; break; }   /* beyond tree size, stop the search */
             largerPtr = nextPtr;
-            if (matchIndex <= btLow) largerPtr=&dummy32; /* beyond tree size, stop the search */
-            matchIndex = (matchIndex <= btLow) ? windowLow : nextPtr[0];
+            matchIndex = nextPtr[0];
         }
     }
 
     *smallerPtr = *largerPtr = 0;
+    return skip+1;
 }
 
 
@@ -443,18 +456,19 @@ size_t ZSTD_HC_insertBtAndFindBestMatch (
 }
 
 
-static void ZSTD_HC_updateTree(ZSTD_HC_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
+static const BYTE* ZSTD_HC_updateTree(ZSTD_HC_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
 {
     const BYTE* const base = zc->base;
     const U32 target = (U32)(ip - base);
     U32 idx = zc->nextToUpdate;
     //size_t dummy;
 
-    for( ; idx < target ; idx++)
-        ZSTD_HC_insertBt1(zc, base+idx, mls, iend, nbCompares);
+    for( ; idx < target ; )
+        idx += ZSTD_HC_insertBt1(zc, base+idx, mls, iend, nbCompares);
         //ZSTD_HC_insertBtAndFindBestMatch(zc, base+idx, iend, &dummy, nbCompares, mls);
 
-    zc->nextToUpdate = target;
+    zc->nextToUpdate = idx;
+    return base + idx;
 }
 
 
@@ -466,7 +480,13 @@ size_t ZSTD_HC_BtFindBestMatch (
                         size_t* offsetPtr,
                         const U32 maxNbAttempts, const U32 mls)
 {
-    ZSTD_HC_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
+    const BYTE* nextToUpdate = ZSTD_HC_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
+    if (nextToUpdate > ip)
+    {
+        /* RLE data */
+        *offsetPtr = 1;
+        return ZSTD_count(ip, ip-1, iLimit);
+    }
     return ZSTD_HC_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls);
 }
 

diff --git a/lib/zstdhc_static.h b/lib/zstdhc_static.h
index 63214a1b5095e5bde63e29b915805ae187ff2576..8f5f903ecd1f1b0eea33b597722bdc17f43e71d3 100644 (file)
--- a/lib/zstdhc_static.h
+++ b/lib/zstdhc_static.h
@@ -106,7 +106,7 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1]
     { 19, 15, 16,  1,  6, ZSTD_HC_fast    },  /* level  2 */
     { 20, 18, 20,  1,  6, ZSTD_HC_fast    },  /* level  3 */
     { 21, 19, 21,  1,  6, ZSTD_HC_fast    },  /* level  4 */
-    { 19, 14, 19,  2,  5, ZSTD_HC_greedy  },  /* level  5 */
+    { 19, 15, 18,  2,  5, ZSTD_HC_greedy  },  /* level  5 */
     { 20, 17, 19,  3,  5, ZSTD_HC_greedy  },  /* level  6 */
     { 21, 17, 20,  3,  5, ZSTD_HC_lazy    },  /* level  7 */
     { 21, 19, 20,  3,  5, ZSTD_HC_lazy    },  /* level  8 */

diff --git a/programs/Makefile b/programs/Makefile
index c3a250e57aeaa33c9b83d449cb0a4b3533e86f5c..ea51bbf7fe3d27e48c7864ea328c46fe75071174 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -30,7 +30,7 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-VERSION?= 0.3.4
+VERSION?= 0.3.5
 
 DESTDIR?=
 PREFIX ?= /usr/local

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 2cb59c76b6e55ce219a9f9c2f1b15abce71936b1..c9946c4deea6578c74d7069a713341f888a0f62e 100644 (file)
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -70,7 +70,7 @@
 **************************************/
 #define COMPRESSOR_NAME "zstd command line interface"
 #ifndef ZSTD_VERSION
-#  define ZSTD_VERSION "v0.3.4"
+#  define ZSTD_VERSION "v0.3.5"
 #endif
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__