Updated fullbench tests

diff --git a/lib/zstd.c b/lib/zstd.c
index 1fc1dcc3c07ea15a0e420c7eebd592e30e7629b7..dcc08bc09f8c1d56f04d79fa552f43959127b0e8 100644 (file)
--- a/lib/zstd.c
+++ b/lib/zstd.c
@@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
     //U32 limit = 4 * BLOCKSIZE;
     //const U32 updateRate = 2 * BLOCKSIZE;
 
-    // Init
+    /*  Init */
     if (ctx->base==NULL) ctx->base = src, ctx->current=0;
-    if (src != ctx->base + ctx->current)   // not contiguous
+    if (src != ctx->base + ctx->current)   /* not contiguous */
     {
             ZSTD_resetCCtx(ctx);
             ctx->base = src;
@@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
         }
         */
 
-        // compress
+        /* compress */
         if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
         cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
         if (cSize == 0)
         {
-            cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
+            cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);   /* block is not compressible */
             if (ZSTD_isError(cSize)) return cSize;
         }
         else
@@ -1084,7 +1084,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
     BYTE* const ostart = dst;
     BYTE* op = ostart;
 
-    // Header
+    /* Header */
     {
         size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
         if(ZSTD_isError(headerSize)) return headerSize;
@@ -1092,7 +1092,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
         maxDstSize -= headerSize;
     }
 
-    // Compression
+    /* Compression */
     {
         size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
         if (ZSTD_isError(cSize)) return cSize;
@@ -1100,7 +1100,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
         maxDstSize -= cSize;
     }
 
-    // Close frame
+    /* Close frame */
     {
         size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize);
         if(ZSTD_isError(endSize)) return endSize;

diff --git a/programs/Makefile b/programs/Makefile
index 8ab270663eb522bb9ba067a364b33044423ca0d5..23a482dec193103d0cfb94e3d4e9a6e17fe599c1 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen
        ./datagen -g256MB | ./zstd32 -v  | ./zstd32 -d > $(VOID)
        ./datagen -g6GB   | ./zstd32 -vq | ./zstd32 -d > $(VOID)
 
-test-fullbench: fullbench
+test-fullbench: fullbench datagen
        ./fullbench -i1
+       ./datagen -P0 -g516K > tmp
+       ./fullbench -i1 tmp
+       @rm tmp
 
-test-fullbench32: fullbench32
+test-fullbench32: fullbench32 datagen
        ./fullbench32 -i1
+       ./datagen -P0 -g516K > tmp
+       ./fullbench32 -i1 tmp
+       @rm tmp
 
 test-fuzzer: fuzzer
        ./fuzzer

diff --git a/programs/datagen.c b/programs/datagen.c
index 0f011baab429a2ff1cc2c13f2b6a216d257d2946..09dfe71eddbd46d08a1b27ab394df50537bfa359 100644 (file)
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -56,6 +56,24 @@
 #endif
 
 
+/**************************************
+*  OS-specific Includes
+**************************************/
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+#  include <fcntl.h>   /* _O_BINARY */
+#  include <io.h>      /* _setmode, _isatty */
+#  ifdef __MINGW32__
+   int _fileno(FILE *stream);   /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
+#  endif
+#  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
+#  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#else
+#  include <unistd.h>  /* isatty */
+#  define SET_BINARY_MODE(file)
+#  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+#endif
+
+
 /**************************************
 *  Constants
 **************************************/
@@ -70,7 +88,6 @@
 #define CDG_SIZE_DEFAULT (64 KB)
 #define CDG_SEED_DEFAULT 0
 #define CDG_COMPRESSIBILITY_DEFAULT 50
-#define CDG_LITDENSITY_DEFAULT 12
 #define PRIME1   2654435761U
 #define PRIME2   2246822519U
 
@@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src)
 {
     U32 rand32 = *src;
     rand32 *= PRIME1;
-    rand32 += PRIME2;
+    rand32 ^= PRIME2;
     rand32  = CDG_rotl32(rand32, 13);
     *src = rand32;
     return rand32;
@@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src)
 
 #define LTSIZE 8192
 #define LTMASK (LTSIZE-1)
-static const char firstChar = '(';
-static const char lastChar = '}';
 static void* CDG_createLiteralDistrib(double ld)
 {
-    char* lt = malloc(LTSIZE);
+    BYTE* lt = malloc(LTSIZE);
     U32 i = 0;
-    char character = '0';
+    BYTE character = '0';
+    BYTE firstChar = '(';
+    BYTE lastChar = '}';
 
+    if (ld==0.0)
+    {
+        character = 0;
+        firstChar = 0;
+        lastChar =255;
+    }
     while (i<LTSIZE)
     {
         U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
@@ -146,8 +169,10 @@ static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba
     U32 pos=1;
     U32 genBlockSize = 128 KB;
     void* ldctx = CDG_createLiteralDistrib(litProba);
+    FILE* fout = stdout;
 
-    /* Build initial prefix */
+    /* init */
+    SET_BINARY_MODE(stdout);
     fullbuff[0] = CDG_genChar(seed, ldctx);
     while (pos<32 KB)
     {
@@ -206,11 +231,8 @@ static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba
             }
         }
 
-        /* output datagen */
-        pos=0;
-        for (;pos+512<=genBlockSize;pos+=512)
-            printf("%512.512s", buff+pos);
-        for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
+        /* output generated data */
+        fwrite(buff, 1, genBlockSize, fout);
         /* Regenerate prefix */
         memcpy(fullbuff, buff + 96 KB, 32 KB);
     }
@@ -238,8 +260,8 @@ static int CDG_usage(char* programName)
 int main(int argc, char** argv)
 {
     int argNb;
-    U32 proba = CDG_COMPRESSIBILITY_DEFAULT;
-    U32 litProba = CDG_LITDENSITY_DEFAULT;
+    double proba = (double)CDG_COMPRESSIBILITY_DEFAULT / 100;
+    double litProba = proba / 3.6;
     U64 size = CDG_SIZE_DEFAULT;
     U32 seed = CDG_SEED_DEFAULT;
     char* programName;
@@ -290,25 +312,28 @@ int main(int argc, char** argv)
                     break;
                 case 'P':
                     argument++;
-                    proba=0;
+                    proba=0.0;
                     while ((*argument>='0') && (*argument<='9'))
                     {
                         proba *= 10;
                         proba += *argument - '0';
                         argument++;
                     }
-                    if (proba>100) proba=100;
+                    if (proba>100.) proba=100.;
+                    proba /= 100.;
+                    litProba = proba / 4.;
                     break;
                 case 'L':
                     argument++;
-                    litProba=0;
+                    litProba=0.;
                     while ((*argument>='0') && (*argument<='9'))
                     {
                         litProba *= 10;
                         litProba += *argument - '0';
                         argument++;
                     }
-                    if (litProba>100) litProba=100;
+                    if (litProba>100.) litProba=100.;
+                    litProba /= 100.;
                     break;
                 case 'v':
                     displayLevel = 4;
@@ -324,9 +349,9 @@ int main(int argc, char** argv)
 
     DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
     DISPLAYLEVEL(3, "Seed = %u \n", seed);
-    if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
+    if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
 
-    CDG_generate(size, &seed, ((double)proba) / 100, ((double)litProba) / 100);
+    CDG_generate(size, &seed, proba, litProba);
 
     return 0;
 }

diff --git a/programs/fullbench.c b/programs/fullbench.c
index 3a9c499313789d18efad2f7ee1ae66f3a424f202..1992df6e0ed30283b55bb301945fdba5844ae5f9 100644 (file)
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
         {
             blockProperties_t bp;
             ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
+            ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp);   // Get first block compressed size
+            if (bp.blockType != bt_compressed)
+            {
+                DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
+                free(dstBuff);
+                free(buff2);
+                return 0;
+            }
             g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3;
             memcpy(buff2, dstBuff+7, g_cSize);
             //srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);   // real speed
@@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
             ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
             ip += 4;   // Jump magic Number
             blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp);   // Get first block compressed size
+            if (bp.blockType != bt_compressed)
+            {
+                DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
+                free(dstBuff);
+                free(buff2);
+                return 0;
+            }
             iend = ip + 3 + blockSize;   // Get end of first block
             ip += 3;   // jump first block header
             ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3;   // jump literal sub block and its header
@@ -450,6 +465,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
     default : ;
     }
 
+    { size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; }     /* warming up memory */
+
     for (loopNb = 1; loopNb <= nbIterations; loopNb++)
     {
         double averageTime;
@@ -457,7 +474,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
         U32 nbRounds=0;
 
         DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
-        { size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; }     /* warming up memory */
 
         milliTime = BMK_GetMilliStart();
         while(BMK_GetMilliStart() == milliTime);