From: inikep Date: Fri, 18 Mar 2016 10:14:58 +0000 (+0100) Subject: Merge remote-tracking branch 'refs/remotes/Cyan4973/dev060' into repcodes X-Git-Tag: v0.6.0^2~17^2~16^2~8^2~15 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5b1586541367a49b106c657e2d92c03788bb6607;p=thirdparty%2Fzstd.git Merge remote-tracking branch 'refs/remotes/Cyan4973/dev060' into repcodes # Conflicts: # lib/zstd_decompress.c # programs/bench.c --- 5b1586541367a49b106c657e2d92c03788bb6607 diff --cc lib/zstd_decompress.c index 201e67516,da9665cd1..bb7ad43d3 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@@ -620,13 -650,13 +650,11 @@@ typedef struct const BYTE* dumpsEnd; } seqState_t; - - static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) { - size_t litLength; - size_t offset; - size_t matchLength; const BYTE* dumps = seqState->dumps; const BYTE* const de = seqState->dumpsEnd; + size_t litLength, offset; /* Literal length */ litLength = FSE_peakSymbol(&(seqState->stateLL)); @@@ -647,67 -677,13 +675,66 @@@ 1 /*fake*/, 1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000, 0x400000, 0x800000, 0x1000000, 0x2000000, 0x4000000, /*fake*/ 1, 1, 1, 1 }; - U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ - U32 nbBits = offsetCode - 1; - if (offsetCode==0) nbBits = 0; /* cmove */ + const U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ + const U32 nbBits = offsetCode ? offsetCode-1 : 0; offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits); if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); - if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset; - if (offsetCode | !litLength) seqState->prevOffset = seq->offset; /* cmove */ +#if ZSTD_REP_NUM == 4 + if (offsetCode==0) offset = 0; + + if (offset < ZSTD_REP_NUM) { + if (!litLength && offset <= 1) + offset = 1-offset; + if (offset != 0) + { + size_t temp = seqState->prevOffset[offset]; + if (offset != 1) { + if (offset == 3) seqState->prevOffset[3] = seqState->prevOffset[2]; + seqState->prevOffset[2] = seqState->prevOffset[1]; + } + offset = temp; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + else + offset = seqState->prevOffset[0]; + } else { + offset -= ZSTD_REP_MOVE; + #if 0 + seqState->prevOffset[3] = seqState->prevOffset[2]; + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + #else + if (kSlotNew < 3) + seqState->prevOffset[3] = seqState->prevOffset[2]; + if (kSlotNew < 2) + seqState->prevOffset[2] = seqState->prevOffset[1]; + if (kSlotNew < 1) + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[kSlotNew] = offset; + #endif + } +#else // ZSTD_REP_NUM == 1 + #if 0 + if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset[0]; /* repcode, cmove */ + else offset -= ZSTD_REP_MOVE; + if (offsetCode | !litLength) seqState->prevOffset[0] = seq->offset; /* cmove */ + #else + if (offsetCode==0) { + if (!litLength) { + offset = seqState->prevOffset[0]; /* repcode, cmove */ + seqState->prevOffset[0] = seq->offset; /* cmove */ + } else + offset = seq->offset; /* repcode, cmove */ + } else { + seqState->prevOffset[0] = seq->offset; /* cmove */ + offset -= ZSTD_REP_MOVE; + } + #endif +#endif FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream)); /* update */ +// printf("offsetCode=%d nbBits=%d offset=%d\n", offsetCode, nbBits, (int)offset); fflush(stdout); } /* Literal length update */ diff --cc programs/bench.c index 40146c10b,d1341fe1c..ff01ca687 --- a/programs/bench.c +++ b/programs/bench.c @@@ -84,11 -83,10 +84,14 @@@ /* ************************************* * Constants ***************************************/ +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "" +#endif + - #define NBLOOPS 3 + #define NBLOOPS 3 + #define TIMELOOP_S 1 + #define ACTIVEPERIOD_S 70 + #define COOLPERIOD_S 10 #define KB *(1 <<10) #define MB *(1 <<20) @@@ -127,15 -126,13 +130,15 @@@ static U32 g_displayLevel = 2; /* 0 /* ************************************* * Benchmark Parameters ***************************************/ - static int nbIterations = NBLOOPS; + static U32 g_nbIterations = NBLOOPS; static size_t g_blockSize = 0; +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + - void BMK_SetNbIterations(int nbLoops) + void BMK_SetNbIterations(unsigned nbLoops) { - nbIterations = nbLoops; - DISPLAYLEVEL(2, "- %i iterations -\n", nbIterations); + g_nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", g_nbIterations); ++ DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations); } void BMK_SetBlockSize(size_t blockSize) @@@ -282,33 -234,40 +252,41 @@@ static int BMK_benchMem(const void* src } } } /* warmimg up memory */ +// int timeloop = additionalParam ? additionalParam : 2500; - int timeloop = 2500; + kSlotNew = additionalParam; RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); /* Bench */ { - int loopNb; + U32 loopNb; - size_t cSize = 0; double fastestC = 100000000., fastestD = 100000000.; double ratio = 0.; U64 crcCheck = 0; + clock_t coolTime = clock(); - DISPLAY("\r%79s\r", ""); + DISPLAYLEVEL(2, "\r%79s\r", ""); - for (loopNb = 1; loopNb <= nbIterations; loopNb++) { + for (loopNb = 1; loopNb <= (g_nbIterations + !g_nbIterations); loopNb++) { int nbLoops; - int milliTime; U32 blockNb; + clock_t clockStart, clockSpan; + clock_t const clockLoop = g_nbIterations ? TIMELOOP_S * CLOCKS_PER_SEC : 10; + + /* overheat protection */ + if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S * CLOCKS_PER_SEC) { + DISPLAY("\rcooling down ... \r"); + BMK_sleep(COOLPERIOD_S); + coolTime = clock(); + } /* Compression */ - DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize); + DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize); - memset(compressedBuffer, 0xE5, maxCompressedSize); + memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ nbLoops = 0; - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliSpan(milliTime) < timeloop) { + clockStart = clock(); + while (clock() == clockStart); + clockStart = clock(); + while (BMK_clockSpan(clockStart) < clockLoop) { ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, ZSTD_getParams(cLevel, MAX(dictBufferSize, largestBlockSize))); for (blockNb=0; blockNb%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.); - DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", ++ DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", + loopNb, displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) ); #if 1 /* Decompression */ @@@ -351,9 -312,12 +331,12 @@@ blockTable[blockNb].resSize = regenSize; } } - milliTime = BMK_GetMilliSpan(milliTime); - if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops; - DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); + clockSpan = BMK_clockSpan(clockStart); + if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops; - DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", ++ DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", + loopNb, displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC), + (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) ); /* CRC Checking */ _findError: @@@ -428,45 -383,20 +411,45 @@@ static void BMK_benchCLevel(void* srcBu const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize) { - if (cLevel < 0) { /* range mode : test all levels from 1 to l */ - int l; - for (l=1; l <= -cLevel; l++) { - BMK_benchMem(srcBuffer, benchedSize, - displayName, l, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); + benchResult_t result, total; + int l; + + const char* pch = strrchr(displayName, '\\'); /* Windows */ + if (!pch) pch = strrchr(displayName, '/'); /* Linux */ + if (pch) displayName = pch+1; + + memset(&result, 0, sizeof(result)); + memset(&total, 0, sizeof(total)); + + if (g_displayLevel == 1 && !additionalParam) - DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); ++ DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); + + if (cLevelLast < cLevel) cLevelLast = cLevel; + + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, additionalParam, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, &result); + if (g_displayLevel == 1) { + if (additionalParam) + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (p=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, additionalParam); + else + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); + total.cSize += result.cSize; + total.cSpeed += result.cSpeed; + total.dSpeed += result.dSpeed; + total.ratio += result.ratio; } - return; } - BMK_benchMem(srcBuffer, benchedSize, - displayName, cLevel, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); + if (g_displayLevel == 1 && cLevelLast > cLevel) + { + total.cSize /= 1+cLevelLast-cLevel; + total.cSpeed /= 1+cLevelLast-cLevel; + total.dSpeed /= 1+cLevelLast-cLevel; + total.ratio /= 1+cLevelLast-cLevel; + DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); + } } static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) @@@ -575,14 -504,14 +557,14 @@@ static void BMK_syntheticTest(int cLeve int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel) + const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) { - double compressibility = (double)g_compressibilityDefault / 100; + double const compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) - BMK_syntheticTest(cLevel, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, additionalParam); return 0; } diff --cc programs/bench.h index 5e6d3b18d,6d62451bd..bc5ffa423 --- a/programs/bench.h +++ b/programs/bench.h @@@ -27,10 -27,10 +27,10 @@@ /* Main function */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel); + const char* dictFileName, int cLevel, int cLevelLast, int additionalParam); /* Set Parameters */ - void BMK_SetNbIterations(int nbLoops); + void BMK_SetNbIterations(unsigned nbLoops); void BMK_SetBlockSize(size_t blockSize); - +void BMK_setNotificationLevel(unsigned level); diff --cc programs/fullbench.c index f6a2321ec,b4afcf112..24cf95d39 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@@ -94,12 -83,13 +83,12 @@@ #define KNUTH 2654435761U #define MAX_MEM (1984 MB) -#define DEFAULT_CHUNKSIZE (4<<20) #define COMPRESSIBILITY_DEFAULT 0.50 - static const size_t sampleSize = 10000000; + static const size_t g_sampleSize = 10000000; - /************************************** + /*_************************************ * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)