Merge remote-tracking branch 'refs/remotes/Cyan4973/dev060' into repcodes

# Conflicts:
#	lib/zstd_decompress.c
#	programs/bench.c

diff --cc lib/zstd_compress.c
Simple merge

diff --cc lib/zstd_decompress.c
index 201e6751695de0ebfc61aea8aa2111335aa8f470,da9665cd12c557e759582a7fd310bad24e95dd02..bb7ad43d3ff91af58b88efc2a6ada0a99749f9bd
--- 1/lib/zstd_decompress.c
--- 2/lib/zstd_decompress.c
+++ b/lib/zstd_decompress.c
@@@ -620,13 -650,13 +650,11 @@@ typedef struct 
      const BYTE* dumpsEnd;
  } seqState_t;
  
 -
 -
  static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls)
  {
-     size_t litLength;
-     size_t offset;
-     size_t matchLength;
      const BYTE* dumps = seqState->dumps;
      const BYTE* const de = seqState->dumpsEnd;
+     size_t litLength, offset;
  
      /* Literal length */
      litLength = FSE_peakSymbol(&(seqState->stateLL));
@@@ -647,67 -677,13 +675,66 @@@
                  1 /*fake*/, 1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80, 0x100,
                  0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000,
                  0x80000, 0x100000, 0x200000, 0x400000, 0x800000, 0x1000000, 0x2000000, 0x4000000, /*fake*/ 1, 1, 1, 1 };
-         U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb));   /* <= maxOff, by table construction */
-         U32 nbBits = offsetCode - 1;
-         if (offsetCode==0) nbBits = 0;   /* cmove */
+         const U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb));   /* <= maxOff, by table construction */
+         const U32 nbBits = offsetCode ? offsetCode-1 : 0;
          offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits);
          if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
 -        if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset;
 -        if (offsetCode | !litLength) seqState->prevOffset = seq->offset;   /* cmove */
 +#if ZSTD_REP_NUM == 4
 +        if (offsetCode==0) offset = 0;
 +        
 +        if (offset < ZSTD_REP_NUM) {
 +            if (!litLength && offset <= 1)
 +                offset = 1-offset;
 +            if (offset != 0)
 +            {
 +                size_t temp = seqState->prevOffset[offset];
 +                if (offset != 1) {
 +                    if (offset == 3) seqState->prevOffset[3] = seqState->prevOffset[2];
 +                    seqState->prevOffset[2] = seqState->prevOffset[1];
 +                }
 +                offset = temp;
 +                seqState->prevOffset[1] = seqState->prevOffset[0];
 +                seqState->prevOffset[0] = offset;
 +            }
 +            else
 +                offset = seqState->prevOffset[0];
 +        } else {
 +            offset -= ZSTD_REP_MOVE;
 +    #if 0
 +            seqState->prevOffset[3] = seqState->prevOffset[2];
 +            seqState->prevOffset[2] = seqState->prevOffset[1];
 +            seqState->prevOffset[1] = seqState->prevOffset[0];
 +            seqState->prevOffset[0] = offset;
 +    #else
 +            if (kSlotNew < 3)
 +                seqState->prevOffset[3] = seqState->prevOffset[2];
 +            if (kSlotNew < 2)
 +                seqState->prevOffset[2] = seqState->prevOffset[1];
 +            if (kSlotNew < 1)
 +                seqState->prevOffset[1] = seqState->prevOffset[0];               
 +            seqState->prevOffset[kSlotNew] = offset;
 +    #endif
 +        }
 +#else // ZSTD_REP_NUM == 1
 +    #if 0
 +        if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset[0];   /* repcode, cmove */
 +        else offset -= ZSTD_REP_MOVE;
 +        if (offsetCode | !litLength) seqState->prevOffset[0] = seq->offset;   /* cmove */
 +    #else
 +        if (offsetCode==0) {
 +            if (!litLength) {
 +                offset = seqState->prevOffset[0];   /* repcode, cmove */
 +                seqState->prevOffset[0] = seq->offset;   /* cmove */
 +            } else
 +                offset = seq->offset;   /* repcode, cmove */
 +        } else {
 +            seqState->prevOffset[0] = seq->offset;   /* cmove */
 +            offset -= ZSTD_REP_MOVE;
 +        }
 +    #endif
 +#endif
          FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream));    /* update */
 +//    printf("offsetCode=%d nbBits=%d offset=%d\n", offsetCode, nbBits, (int)offset); fflush(stdout);
      }
  
      /* Literal length update */

diff --cc lib/zstd_internal.h
Simple merge

diff --cc programs/bench.c
index 40146c10b3c0758db48606fdf216e27d305d46dc,d1341fe1c475ebd488ea7da08e5005e48e60d494..ff01ca687edf48ede93f5ed26203df201f8182fa
--- 1/programs/bench.c
--- 2/programs/bench.c
+++ b/programs/bench.c
@@@ -84,11 -83,10 +84,14 @@@
  /* *************************************
  *  Constants
  ***************************************/
- #define NBLOOPS    3
 +#ifndef ZSTD_VERSION
 +#  define ZSTD_VERSION ""
 +#endif
 +
+ #define NBLOOPS          3
+ #define TIMELOOP_S       1
+ #define ACTIVEPERIOD_S  70
+ #define COOLPERIOD_S    10
  
  #define KB *(1 <<10)
  #define MB *(1 <<20)
@@@ -127,15 -126,13 +130,15 @@@ static U32 g_displayLevel = 2;   /* 0 
  /* *************************************
  *  Benchmark Parameters
  ***************************************/
- static int nbIterations = NBLOOPS;
+ static U32 g_nbIterations = NBLOOPS;
  static size_t g_blockSize = 0;
  
- void BMK_SetNbIterations(int nbLoops)
 +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 +
+ void BMK_SetNbIterations(unsigned nbLoops)
  {
-     nbIterations = nbLoops;
-     DISPLAYLEVEL(2, "- %i iterations -\n", nbIterations);
+     g_nbIterations = nbLoops;
 -    DISPLAY("- %i iterations -\n", g_nbIterations);
++    DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations);
  }
  
  void BMK_SetBlockSize(size_t blockSize)
@@@ -282,33 -234,40 +252,41 @@@ static int BMK_benchMem(const void* src
      }   }   }
  
      /* warmimg up memory */
-     int timeloop = 2500;
 +//    int timeloop = additionalParam ? additionalParam : 2500;
 +    kSlotNew = additionalParam;
      RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
  
      /* Bench */
      {
-         int loopNb;
+         U32 loopNb;
 -        size_t cSize = 0;
          double fastestC = 100000000., fastestD = 100000000.;
          double ratio = 0.;
          U64 crcCheck = 0;
+         clock_t coolTime = clock();
  
 -        DISPLAY("\r%79s\r", "");
 +        DISPLAYLEVEL(2, "\r%79s\r", "");
-         for (loopNb = 1; loopNb <= nbIterations; loopNb++) {
+         for (loopNb = 1; loopNb <= (g_nbIterations + !g_nbIterations); loopNb++) {
              int nbLoops;
-             int milliTime;
              U32 blockNb;
+             clock_t clockStart, clockSpan;
+             clock_t const clockLoop = g_nbIterations ? TIMELOOP_S * CLOCKS_PER_SEC : 10;
+ 
+             /* overheat protection */
+             if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S * CLOCKS_PER_SEC) {
+                 DISPLAY("\rcooling down ...    \r");
+                 BMK_sleep(COOLPERIOD_S);
+                 coolTime = clock();
+             }
  
              /* Compression */
 -            DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize);
 +            DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize);
-             memset(compressedBuffer, 0xE5, maxCompressedSize);
+             memset(compressedBuffer, 0xE5, maxCompressedSize);  /* warm up and erase result buffer */
  
              nbLoops = 0;
-             milliTime = BMK_GetMilliStart();
-             while (BMK_GetMilliStart() == milliTime);
-             milliTime = BMK_GetMilliStart();
-             while (BMK_GetMilliSpan(milliTime) < timeloop) {
+             clockStart = clock();
+             while (clock() == clockStart);
+             clockStart = clock();
+             while (BMK_clockSpan(clockStart) < clockLoop) {
                  ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, ZSTD_getParams(cLevel, MAX(dictBufferSize, largestBlockSize)));
                  for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                      size_t rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
@@@ -324,9 -283,11 +302,11 @@@
              cSize = 0;
              for (blockNb=0; blockNb<nbBlocks; blockNb++)
                  cSize += blockTable[blockNb].cSize;
-             if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
+             if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops;
              ratio = (double)srcSize / (double)cSize;
-             DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
 -            DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
++            DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
+                     loopNb, displayName, (U32)srcSize, (U32)cSize, ratio,
+                     (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) );
  
  #if 1
              /* Decompression */
@@@ -351,9 -312,12 +331,12 @@@
                      blockTable[blockNb].resSize = regenSize;
              }   }
  
-             milliTime = BMK_GetMilliSpan(milliTime);
-             if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
-             DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
+             clockSpan = BMK_clockSpan(clockStart);
+             if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops;
 -            DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
++            DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
+                     loopNb, displayName, (U32)srcSize, (U32)cSize, ratio,
+                     (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC),
+                     (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) );
  
              /* CRC Checking */
  _findError:
@@@ -428,45 -383,20 +411,45 @@@ static void BMK_benchCLevel(void* srcBu
                              const size_t* fileSizes, unsigned nbFiles,
                              const void* dictBuffer, size_t dictBufferSize)
  {
 -    if (cLevel < 0) {  /* range mode : test all levels from 1 to l */
 -        int l;
 -        for (l=1; l <= -cLevel; l++) {
 -            BMK_benchMem(srcBuffer, benchedSize,
 -                         displayName, l,
 -                         fileSizes, nbFiles,
 -                         dictBuffer, dictBufferSize);
 +    benchResult_t result, total;
 +    int l;
 +
 +    const char* pch = strrchr(displayName, '\\'); /* Windows */
 +    if (!pch) pch = strrchr(displayName, '/'); /* Linux */
 +    if (pch) displayName = pch+1;
 +
 +    memset(&result, 0, sizeof(result));
 +    memset(&total, 0, sizeof(total));
 +
 +    if (g_displayLevel == 1 && !additionalParam)
-         DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10));
++        DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
 +
 +    if (cLevelLast < cLevel) cLevelLast = cLevel;
 +
 +    for (l=cLevel; l <= cLevelLast; l++) {           
 +        BMK_benchMem(srcBuffer, benchedSize,
 +                     displayName, l, additionalParam,
 +                     fileSizes, nbFiles,
 +                     dictBuffer, dictBufferSize, &result);
 +        if (g_displayLevel == 1) {
 +            if (additionalParam)
 +                DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s  %s (p=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, additionalParam);
 +            else
 +                DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s  %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName);
 +            total.cSize += result.cSize;
 +            total.cSpeed += result.cSpeed;
 +            total.dSpeed += result.dSpeed;
 +            total.ratio += result.ratio;
          }
 -        return;
      }
 -    BMK_benchMem(srcBuffer, benchedSize,
 -                 displayName, cLevel,
 -                 fileSizes, nbFiles,
 -                 dictBuffer, dictBufferSize);
 +    if (g_displayLevel == 1 && cLevelLast > cLevel)
 +    {
 +        total.cSize /= 1+cLevelLast-cLevel;
 +        total.cSpeed /= 1+cLevelLast-cLevel;
 +        total.dSpeed /= 1+cLevelLast-cLevel;
 +        total.ratio /= 1+cLevelLast-cLevel;
 +        DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s  %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName);            
 +    }
  }
  
  static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
@@@ -575,14 -504,14 +557,14 @@@ static void BMK_syntheticTest(int cLeve
  
  
  int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
 -                   const char* dictFileName, int cLevel)
 +                   const char* dictFileName, int cLevel, int cLevelLast, int additionalParam)
  {
-     double compressibility = (double)g_compressibilityDefault / 100;
+     double const compressibility = (double)g_compressibilityDefault / 100;
  
      if (nbFiles == 0)
 -        BMK_syntheticTest(cLevel, compressibility);
 +        BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility);
      else
 -        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel);
 +        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, additionalParam);
      return 0;
  }
  

diff --cc programs/bench.h
index 5e6d3b18dfe855a5a327cc20c9ddabacb3e31b6e,6d62451bda2023949441e915c8eb0eb4972ae312..bc5ffa423e99522d3de6ae92868fcf4f44dc4f20
--- 1/programs/bench.h
--- 2/programs/bench.h
+++ b/programs/bench.h
@@@ -27,10 -27,10 +27,10 @@@
  
  /* Main function */
  int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
 -                   const char* dictFileName, int cLevel);
 +                   const char* dictFileName, int cLevel, int cLevelLast, int additionalParam);
  
  /* Set Parameters */
- void BMK_SetNbIterations(int nbLoops);
+ void BMK_SetNbIterations(unsigned nbLoops);
  void BMK_SetBlockSize(size_t blockSize);
 -
 +void BMK_setNotificationLevel(unsigned level);
  

diff --cc programs/fullbench.c
index f6a2321ec50f3f6100d9cfef1cab45d708b35da7,b4afcf11286e21ce0fa42d5dad9be480abecd6c6..24cf95d3954dd48ace50a8b1339ea136fc7a9c2e
--- 1/programs/fullbench.c
--- 2/programs/fullbench.c
+++ b/programs/fullbench.c
@@@ -94,12 -83,13 +83,12 @@@
  
  #define KNUTH      2654435761U
  #define MAX_MEM    (1984 MB)
 -#define DEFAULT_CHUNKSIZE   (4<<20)
  
  #define COMPRESSIBILITY_DEFAULT 0.50
- static const size_t sampleSize = 10000000;
+ static const size_t g_sampleSize = 10000000;
  
  
- /**************************************
+ /*_************************************
  *  Macros
  **************************************/
  #define DISPLAY(...)  fprintf(stderr, __VA_ARGS__)

diff --cc programs/zstdcli.c
Simple merge