From: inikep Date: Tue, 22 Mar 2016 13:38:34 +0000 (+0100) Subject: Merge remote-tracking branch 'refs/remotes/Cyan4973/dev060' into dev X-Git-Tag: v0.6.0^2~17^2~17^2~16 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2872b6f01f86f7f056245ff3a55799017d767f69;p=thirdparty%2Fzstd.git Merge remote-tracking branch 'refs/remotes/Cyan4973/dev060' into dev # Conflicts: # lib/zstd_compress.c # programs/bench.c --- 2872b6f01f86f7f056245ff3a55799017d767f69 diff --cc lib/zstd_compress.c index 2ee25cc93,cfb2519ad..266999ca7 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@@ -266,15 -267,14 +268,15 @@@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx memcpy(dstCCtx->headerBuffer , srcCCtx->headerBuffer, srcCCtx->hbSize); /* copy dictionary pointers */ - dstCCtx->nextToUpdate= srcCCtx->nextToUpdate; - dstCCtx->nextToUpdate3 = srcCCtx->nextToUpdate3; - dstCCtx->nextSrc = srcCCtx->nextSrc; - dstCCtx->base = srcCCtx->base; - dstCCtx->dictBase = srcCCtx->dictBase; - dstCCtx->dictLimit = srcCCtx->dictLimit; - dstCCtx->lowLimit = srcCCtx->lowLimit; - dstCCtx->loadedDictEnd = srcCCtx->loadedDictEnd; + dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; + dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3; + dstCCtx->nextSrc = srcCCtx->nextSrc; + dstCCtx->base = srcCCtx->base; + dstCCtx->dictBase = srcCCtx->dictBase; + dstCCtx->dictLimit = srcCCtx->dictLimit; + dstCCtx->lowLimit = srcCCtx->lowLimit; + dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; + dstCCtx->additionalParam = srcCCtx->additionalParam; /* copy entropy tables */ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; diff --cc programs/bench.c index e0db60e5b,c74c03df2..5b2e5644a --- a/programs/bench.c +++ b/programs/bench.c @@@ -84,12 -83,10 +84,14 @@@ /* ************************************* * Constants ***************************************/ +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "" +#endif + - #define NBLOOPS 3 - #define TIMELOOP 2500 + #define NBLOOPS 3 + #define TIMELOOP_S 1 + #define ACTIVEPERIOD_S 70 + #define COOLPERIOD_S 10 #define KB *(1 <<10) #define MB *(1 <<20) @@@ -128,15 -126,13 +130,15 @@@ static U32 g_displayLevel = 2; /* 0 /* ************************************* * Benchmark Parameters ***************************************/ - static int nbIterations = NBLOOPS; + static U32 g_nbIterations = NBLOOPS; static size_t g_blockSize = 0; +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + - void BMK_SetNbIterations(int nbLoops) + void BMK_SetNbIterations(unsigned nbLoops) { - nbIterations = nbLoops; - DISPLAYLEVEL(2, "- %i iterations -\n", nbIterations); + g_nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", g_nbIterations); ++ DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations); } void BMK_SetBlockSize(size_t blockSize) @@@ -245,14 -198,10 +213,11 @@@ static int BMK_benchMem(const void* src ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_DCtx* refDCtx = ZSTD_createDCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); - U64 crcOrig = XXH64(srcBuffer, srcSize, 0); - U32 nbBlocks = 0; - size_t cSize = 0; - - /* init */ - if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ + - /* Memory allocation & restrictions */ + U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); + U32 nbBlocks; + + /* checks */ if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx) EXM_THROW(31, "not enough memory"); @@@ -290,40 -238,49 +255,49 @@@ double fastestC = 100000000., fastestD = 100000000.; double ratio = 0.; U64 crcCheck = 0; + clock_t coolTime = clock(); + U32 testNb; - DISPLAY("\r%79s\r", ""); + DISPLAYLEVEL(2, "\r%79s\r", ""); - for (loopNb = 1; loopNb <= nbIterations; loopNb++) { + for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) { int nbLoops; - int milliTime; - U32 blockNb; + clock_t clockStart, clockSpan; + clock_t const clockLoop = g_nbIterations ? TIMELOOP_S * CLOCKS_PER_SEC : 10; + + /* overheat protection */ + if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S * CLOCKS_PER_SEC) { + DISPLAY("\rcooling down ... \r"); + BMK_sleep(COOLPERIOD_S); + coolTime = clock(); + } /* Compression */ - DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize); - memset(compressedBuffer, 0xE5, maxCompressedSize); - - nbLoops = 0; - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliSpan(milliTime) < TIMELOOP) { - ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, ZSTD_getParams(cLevel, MAX(dictBufferSize, largestBlockSize))); - DISPLAY("%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize); ++ DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize); + memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + + clockStart = clock(); + while (clock() == clockStart); + clockStart = clock(); + + for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { + U32 blockNb; + ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel); for (blockNb=0; blockNb%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.); - DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", ++ DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", + testNb, displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) ); #if 1 /* Decompression */ @@@ -348,9 -305,12 +322,12 @@@ blockTable[blockNb].resSize = regenSize; } } - milliTime = BMK_GetMilliSpan(milliTime); - if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops; - DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); + clockSpan = BMK_clockSpan(clockStart); + if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops; - DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", ++ DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", + testNb, displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC), + (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) ); /* CRC Checking */ _findError: @@@ -376,21 -336,11 +353,18 @@@ printf("no difference detected\n"); } } break; - } + } /* if (crcOrig!=crcCheck) */ #endif - } + } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ - DISPLAY("%2i#\n", cLevel); + - if (crcOrig == crcCheck) - { - DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); ++ if (crcOrig == crcCheck) { + result->ratio = ratio; + result->cSize = cSize; - result->cSpeed = (double)srcSize / fastestC / 1000.; - result->dSpeed = (double)srcSize / fastestD / 1000.; ++ result->cSpeed = (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC); ++ result->dSpeed = (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC); + } - else - DISPLAYLEVEL(2, "%2i-\n", cLevel); - } ++ DISPLAYLEVEL(2, "%2i#\n", cLevel); + } /* Bench */ /* clean up */ free(compressedBuffer); @@@ -425,45 -376,20 +400,45 @@@ static void BMK_benchCLevel(void* srcBu const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize) { - if (cLevel < 0) { /* range mode : test all levels from 1 to l */ - int l; - for (l=1; l <= -cLevel; l++) { - BMK_benchMem(srcBuffer, benchedSize, - displayName, l, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); + benchResult_t result, total; + int l; + + const char* pch = strrchr(displayName, '\\'); /* Windows */ + if (!pch) pch = strrchr(displayName, '/'); /* Linux */ + if (pch) displayName = pch+1; + + memset(&result, 0, sizeof(result)); + memset(&total, 0, sizeof(total)); + + if (g_displayLevel == 1 && !additionalParam) - DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); ++ DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); + + if (cLevelLast < cLevel) cLevelLast = cLevel; + + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, additionalParam, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, &result); + if (g_displayLevel == 1) { + if (additionalParam) + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (p=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, additionalParam); + else + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); + total.cSize += result.cSize; + total.cSpeed += result.cSpeed; + total.dSpeed += result.dSpeed; + total.ratio += result.ratio; } - return; } - BMK_benchMem(srcBuffer, benchedSize, - displayName, cLevel, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); + if (g_displayLevel == 1 && cLevelLast > cLevel) + { + total.cSize /= 1+cLevelLast-cLevel; + total.cSpeed /= 1+cLevelLast-cLevel; + total.dSpeed /= 1+cLevelLast-cLevel; + total.ratio /= 1+cLevelLast-cLevel; + DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); + } } static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) @@@ -572,14 -497,14 +546,14 @@@ static void BMK_syntheticTest(int cLeve int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel) + const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) { - double compressibility = (double)g_compressibilityDefault / 100; + double const compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) - BMK_syntheticTest(cLevel, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, additionalParam); return 0; } diff --cc programs/bench.h index 5e6d3b18d,6d62451bd..bc5ffa423 --- a/programs/bench.h +++ b/programs/bench.h @@@ -27,10 -27,10 +27,10 @@@ /* Main function */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel); + const char* dictFileName, int cLevel, int cLevelLast, int additionalParam); /* Set Parameters */ - void BMK_SetNbIterations(int nbLoops); + void BMK_SetNbIterations(unsigned nbLoops); void BMK_SetBlockSize(size_t blockSize); - +void BMK_setNotificationLevel(unsigned level); diff --cc programs/fullbench.c index f6a2321ec,b4afcf112..24cf95d39 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@@ -94,12 -83,13 +83,12 @@@ #define KNUTH 2654435761U #define MAX_MEM (1984 MB) -#define DEFAULT_CHUNKSIZE (4<<20) #define COMPRESSIBILITY_DEFAULT 0.50 - static const size_t sampleSize = 10000000; + static const size_t g_sampleSize = 10000000; - /************************************** + /*_************************************ * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) diff --cc programs/zstdcli.c index c986caee3,86d6bff78..a14fe26de --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@@ -331,17 -325,11 +333,17 @@@ int main(int argCount, const char** arg dictSelect *= 10, dictSelect += *argument++ - '0'; break; - /* Pause at the end (hidden option) */ - case 'p': main_pause=1; argument++; break; - + /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ + case 'p': argument++; + if ((*argument>='0') && (*argument<='9')) { + additionalParam = 0; + while ((*argument >= '0') && (*argument <= '9')) + additionalParam *= 10, additionalParam += *argument++ - '0'; + continue; + } + main_pause=1; break; /* unknown command */ - default : return badusage(programName); + default : CLEAN_RETURN(badusage(programName)); } } continue;