From: Yann Collet Date: Mon, 26 Oct 2015 14:45:58 +0000 (+0100) Subject: skip slow conf faster X-Git-Tag: zstd-0.3.0^2~24 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=786f5b554a7b6d810af800ef640bde11089f6592;p=thirdparty%2Fzstd.git skip slow conf faster --- diff --git a/lib/zstdhc.c b/lib/zstdhc.c index 71ff7a160..1671f4265 100644 --- a/lib/zstdhc.c +++ b/lib/zstdhc.c @@ -64,31 +64,6 @@ static const U32 ZSTD_HC_compressionLevel_default = 9; #define BLOCKSIZE (128 KB) /* define, for static allocation */ #define WORKPLACESIZE (BLOCKSIZE*3) -#define MAX_CLEVEL 20 -static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[MAX_CLEVEL] = { - /* W, C, H, S */ - { 19, 18, 17 , 1}, - { 19, 18, 17 , 1}, - { 19, 18, 17 , 2}, - { 19, 18, 17 , 3}, - { 19, 18, 17 , 4}, - { 19, 18, 17 , 5}, - { 19, 18, 17 , 6}, - { 19, 18, 17 , 7}, - { 19, 18, 17 , 8}, - { 19, 18, 17 , 9}, - { 19, 18, 17 ,10}, - { 19, 18, 17 ,11}, - { 19, 18, 17 ,12}, - { 19, 18, 17 ,13}, - { 19, 18, 17 ,14}, - { 19, 18, 17 ,15}, - { 19, 18, 17 ,16}, - { 19, 18, 17 ,17}, - { 19, 18, 17 ,18}, - { 19, 18, 17 ,19}, -}; - struct ZSTD_HC_CCtx_s { const BYTE* end; /* next block here to continue on current prefix */ @@ -461,7 +436,7 @@ size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx, size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, unsigned compressionLevel, const void* src) { if (compressionLevel==0) compressionLevel = ZSTD_HC_compressionLevel_default; - if (compressionLevel > MAX_CLEVEL) compressionLevel = MAX_CLEVEL; + if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL; return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[compressionLevel], src); } @@ -496,8 +471,8 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx, size_t ZSTD_HC_compressCCtx (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel) { - if (compressionLevel==0) compressionLevel = ZSTD_HC_compressionLevel_default; - if (compressionLevel > MAX_CLEVEL) compressionLevel = MAX_CLEVEL; + if (compressionLevel==0) return ZSTD_compress(dst, maxDstSize, src, srcSize); /* fast mode */ + if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL; return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[compressionLevel]); } diff --git a/lib/zstdhc_static.h b/lib/zstdhc_static.h index 80935ffcf..26d910fad 100644 --- a/lib/zstdhc_static.h +++ b/lib/zstdhc_static.h @@ -65,6 +65,36 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx, ZSTD_HC_parameters params); +/* ************************************* +* Pre-defined compression levels +***************************************/ +#define ZSTD_HC_MAX_CLEVEL 20 +static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = { + /* W, C, H, S */ + { 18, 4, 12, 1 }, /* 0 - should never be used */ + { 18, 10, 14, 1 }, /* 1 */ + { 18, 13, 15, 1 }, /* 2 */ + { 20, 18, 19, 2 }, /* 3 */ + { 21, 19, 20, 3 }, /* 4 */ + { 21, 19, 21, 4 }, /* 5 */ + { 21, 19, 21, 5 }, /* 6 */ + { 21, 21, 21, 5 }, /* 7 */ + { 21, 21, 21, 6 }, /* 8 */ + { 21, 21, 21, 7 }, /* 9 */ + { 21, 21, 21, 8 }, /*10 */ + { 21, 21, 21, 9 }, /*11 */ + { 21, 21, 21, 10 }, /*12 */ + { 21, 21, 21, 11 }, /*13 */ + { 21, 21, 21, 13 }, /*14 */ + { 21, 21, 21, 17 }, /*15 */ + { 22, 22, 22, 18 }, /*16 */ + { 23, 23, 23, 19 }, /*17 */ + { 24, 24, 24, 20 }, /*18 */ + { 25, 25, 25, 21 }, /*19 */ + { 26, 26, 26, 25 }, /*20 - ultra-slow */ +}; + + #if defined (__cplusplus) } #endif diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 1d221ab12..24eba9253 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -100,6 +100,10 @@ #define COMPRESSIBILITY_DEFAULT 0.50 static const size_t sampleSize = 10000000; +static const int g_grillDuration = 30000000; /* about 8.5 hours */ +static const int g_maxParamTime = 15000; /* 15 sec */ +static const int g_maxVariationTime = 60000; /* 60 sec */ +static const int g_maxNbVariations = 32; /************************************** * Macros @@ -113,6 +117,7 @@ static const size_t sampleSize = 10000000; static U32 nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; static U32 g_blockSize = 0; +static U32 g_rand = 1; void BMK_SetNbIterations(int nbLoops) { @@ -296,6 +301,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, double fastestC = 100000000., fastestD = 100000000.; double ratio = 0.; U64 crcCheck = 0; + const int startTime =BMK_GetMilliStart(); DISPLAY("\r%79s\r", ""); for (loopNb = 1; loopNb <= nbIterations; loopNb++) @@ -303,6 +309,10 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, int nbLoops; int milliTime; U32 blockNb; + const int totalTime = BMK_GetMilliSpan(startTime); + + /* early break (slow params) */ + if (totalTime > g_maxParamTime) break; /* Compression */ DISPLAY("%1u-W%02uC%02uH%02uS%02u : %9u ->\r", loopNb, Wlog, Clog, Hlog, Slog, (U32)srcSize); @@ -316,9 +326,9 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, { for (blockNb=0; blockNb W%02uC%02uH%02uS%02u \n", + fprintf(f, + "level %2u: R:%5.3f at %5.1f MB/s => W%02uC%02uH%02uS%02u \n", cLevel, (double)srcSize / result.cSize, (double)result.cSpeed / 1000., params.windowLog, params.chainLog, params.hashLog, params.searchLog); + } -#define CLEVEL_MAX 21 -static U32 g_cSpeedTarget[CLEVEL_MAX] = { 300000, 200000, 150000, 100000, 70000, 50000, 35000, 25000, 15000, 10000, /* 0 - 9 */ - 7000, 5000, 3500, 2500, 1500, 1000, 700, 500, 350, 250, /* 10 - 19 */ - 0 }; /* 20 */ +static U32 g_cSpeedTarget[ZSTD_HC_MAX_CLEVEL+1] = { + 300000, 200000, 150000, 100000, 70000, 50000, 35000, 25000, 15000, 10000, /* 0 - 9 */ + 7000, 5000, 3500, 2500, 1500, 1000, 700, 500, 350, 250, /* 10 - 19 */ + 0 }; /* 20 */ typedef struct { BMK_result_t result; ZSTD_HC_parameters params; } winnerInfo_t; -static void BMK_printWinners(const winnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize) { int cLevel; DISPLAY("\nSelected configurations :\n"); - for (cLevel=0; cLevel < CLEVEL_MAX; cLevel++) - BMK_printWinner(cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); + for (cLevel=0; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++) + BMK_printWinner(stdout, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); + fseek(f, 0, SEEK_SET); + for (cLevel=0; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++) + BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); } -#define WINDOWLOG_MAX 26 -#define WINDOWLOG_MIN 17 -#define CHAINLOG_MAX WINDOWLOG_MAX -#define CHAINLOG_MIN 4 -#define HASHLOG_MAX 28 -#define HASHLOG_MIN 4 -#define SEARCHLOG_MAX (CHAINLOG_MAX-1) -#define SEARCHLOG_MIN 1 - -#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) - -static void playAround(winnerInfo_t* winners, ZSTD_HC_parameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_HC_CCtx* ctx) +static int BMK_seed(winnerInfo_t* winners, ZSTD_HC_parameters params, + const void* srcBuffer, size_t srcSize, + ZSTD_HC_CCtx* ctx) { BMK_result_t testResult; int better = 0; int cLevel; - static BYTE g_alreadyTested[WINDOWLOG_MAX+1-WINDOWLOG_MIN] - [CHAINLOG_MAX+1-CHAINLOG_MIN] - [HASHLOG_MAX+1-HASHLOG_MIN] - [SEARCHLOG_MAX+1-SEARCHLOG_MIN] = {}; /* init to zero */ - static U32 g_rand = 1; - - /* exclude faster already played params */ - if (FUZ_rand(&g_rand) & ((1 << g_alreadyTested[params.windowLog-WINDOWLOG_MIN] - [params.chainLog-CHAINLOG_MIN] - [params.hashLog-HASHLOG_MIN] - [params.searchLog-SEARCHLOG_MIN]++)-1)) - return; - BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); - for (cLevel = 0; cLevel < CLEVEL_MAX; cLevel++) + for (cLevel = 0; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++) { if ( (testResult.cSpeed > g_cSpeedTarget[cLevel]) && ((winners[cLevel].result.cSize==0) || (winners[cLevel].result.cSize > testResult.cSize)) ) @@ -454,110 +445,169 @@ static void playAround(winnerInfo_t* winners, ZSTD_HC_parameters params, better = 1; winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(cLevel, testResult, params, srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, srcSize); } } - if (!better) return; + return better; +} - BMK_printWinners(winners, srcSize); +#define WINDOWLOG_MAX 26 +#define WINDOWLOG_MIN 17 +#define CHAINLOG_MAX WINDOWLOG_MAX +#define CHAINLOG_MIN 4 +#define HASHLOG_MAX 28 +#define HASHLOG_MIN 4 +#define SEARCHLOG_MAX (CHAINLOG_MAX-1) +#define SEARCHLOG_MIN 1 - if (params.searchLog > SEARCHLOG_MIN) - { - params.searchLog--; - playAround(winners, params, srcBuffer, srcSize, ctx); - params.searchLog++; - } +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) - if (params.hashLog < HASHLOG_MAX) - { - params.hashLog++; - playAround(winners, params, srcBuffer, srcSize, ctx); - params.hashLog--; - } +static BYTE g_alreadyTested[WINDOWLOG_MAX+1-WINDOWLOG_MIN] + [CHAINLOG_MAX+1-CHAINLOG_MIN] + [HASHLOG_MAX+1-HASHLOG_MIN] + [SEARCHLOG_MAX+1-SEARCHLOG_MIN] = {}; /* init to zero */ - if (params.chainLog > CHAINLOG_MIN) - { - params.chainLog--; - playAround(winners, params, srcBuffer, srcSize, ctx); - params.chainLog++; - } +#define NB_TESTS_PLAYED(p) \ + g_alreadyTested[p.windowLog-WINDOWLOG_MIN][p.chainLog-CHAINLOG_MIN][p.hashLog-HASHLOG_MIN][p.searchLog-SEARCHLOG_MIN] - if (params.windowLog > MAX(WINDOWLOG_MIN, params.chainLog)) - { - params.windowLog--; - playAround(winners, params, srcBuffer, srcSize, ctx); - params.windowLog++; - } - if (params.windowLog < WINDOWLOG_MAX) +static void playAround(FILE* f, winnerInfo_t* winners, + ZSTD_HC_parameters params, + const void* srcBuffer, size_t srcSize, + ZSTD_HC_CCtx* ctx) +{ + int nbVariations = 0; + const int startTime = BMK_GetMilliStart(); + while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) { - params.windowLog++; - playAround(winners, params, srcBuffer, srcSize, ctx); - params.windowLog--; - } + ZSTD_HC_parameters p = params; + U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; + if (nbVariations++ > g_maxNbVariations) break; - if (params.chainLog < params.windowLog) - { - params.chainLog++; - playAround(winners, params, srcBuffer, srcSize, ctx); - params.chainLog--; - } + for (; nbChanges; nbChanges--) + { + const U32 changeID = FUZ_rand(&g_rand) & 7; + switch(changeID) + { + case 0: + p.chainLog++; break; + case 1: + p.chainLog--; break; + case 2: + p.hashLog++; break; + case 3: + p.hashLog--; break; + case 4: + p.searchLog++; break; + case 5: + p.searchLog--; break; + case 6: + p.windowLog++; break; + case 7: + p.windowLog--; break; + } + } - if (params.hashLog > HASHLOG_MIN) - { - params.hashLog--; - playAround(winners, params, srcBuffer, srcSize, ctx); - params.hashLog++; + /* validate new conf */ + if (p.windowLog > WINDOWLOG_MAX) continue; + if (p.windowLog < MAX(WINDOWLOG_MIN, p.chainLog)) continue; + if (p.chainLog > p.windowLog) continue; + if (p.chainLog < CHAINLOG_MIN) continue; + if (p.hashLog > HASHLOG_MAX) continue; + if (p.hashLog < HASHLOG_MIN) continue; + if (p.searchLog > p.chainLog) continue; + if (p.searchLog < SEARCHLOG_MIN) continue; + + /* exclude faster if already played params */ + if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) continue; + + /* test */ + NB_TESTS_PLAYED(p)++; + if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx)) continue; + + /* improvement found => search more */ + BMK_printWinners(f, winners, srcSize); + playAround(f, winners, p, srcBuffer, srcSize, ctx); } - if (params.searchLog < params.chainLog) +} + + +static void BMK_selectRandomStart( + FILE* f, winnerInfo_t* winners, + const void* srcBuffer, size_t srcSize, + ZSTD_HC_CCtx* ctx) +{ + U32 id = FUZ_rand(&g_rand) % (ZSTD_HC_MAX_CLEVEL+1); + if (id==0) { - params.searchLog++; - playAround(winners, params, srcBuffer, srcSize, ctx); - params.searchLog--; + /* totally random entry */ + ZSTD_HC_parameters p; + p.chainLog = FUZ_rand(&g_rand) % (CHAINLOG_MAX+1 - CHAINLOG_MIN) + CHAINLOG_MIN; + p.hashLog = FUZ_rand(&g_rand) % (HASHLOG_MAX+1 - HASHLOG_MIN) + HASHLOG_MIN; + p.searchLog = FUZ_rand(&g_rand) % (SEARCHLOG_MAX+1 - SEARCHLOG_MIN) + SEARCHLOG_MIN; + p.windowLog = FUZ_rand(&g_rand) % (WINDOWLOG_MAX+1 - WINDOWLOG_MIN) + WINDOWLOG_MIN; + playAround(f, winners, p, srcBuffer, srcSize, ctx); } - + else + playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx); } + +static const ZSTD_HC_parameters* seedParams = ZSTD_HC_defaultParameters; + static void BMK_benchMem(void* srcBuffer, size_t srcSize) { ZSTD_HC_CCtx* ctx = ZSTD_HC_createCCtx(); ZSTD_HC_parameters params; - winnerInfo_t winners[CLEVEL_MAX]; + winnerInfo_t winners[ZSTD_HC_MAX_CLEVEL+1]; BMK_result_t testResult; + int i; + const char* rfName = "grillResults.txt"; + FILE* f; /* init */ memset(winners, 0, sizeof(winners)); + f = fopen(rfName, "w"); + if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } /* baseline config for level 9 */ params.windowLog = 19; - params.chainLog = 18; - params.hashLog = 17; + params.chainLog = 19; + params.hashLog = 19; params.searchLog = 9; BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); /* establish speed objectives (relative to current platform) */ + g_cSpeedTarget[9] = (testResult.cSpeed * 15) >> 4; + g_cSpeedTarget[1] = g_cSpeedTarget[9] << 4; + g_cSpeedTarget[0] = (g_cSpeedTarget[1] * 181) >> 7; /* sqrt2 */ + for (i=2; i> 8; + + /* populate initial solution */ + BMK_seed(winners, params, srcBuffer, srcSize, ctx); + for (i=1; i<=ZSTD_HC_MAX_CLEVEL; i++) + BMK_seed(winners, seedParams[i], srcBuffer, srcSize, ctx); + BMK_printWinners(f, winners, srcSize); + + /* start tests */ { - int i; - g_cSpeedTarget[9] = (testResult.cSpeed * 15) >> 4; - g_cSpeedTarget[1] = g_cSpeedTarget[9] << 4; - g_cSpeedTarget[0] = (g_cSpeedTarget[1] * 181) >> 7; /* sqrt2 */ - for (i=2; i<(CLEVEL_MAX-1); i++) /* last level : pure compression */ - g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 181) >> 8; + const int milliStart = BMK_GetMilliStart(); + int mLength; + do + { + BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx); + mLength = BMK_GetMilliSpan(milliStart); + } while (mLength < g_grillDuration); } - /* start from fast conf */ - params.windowLog = WINDOWLOG_MIN; - params.chainLog = CHAINLOG_MIN; - params.hashLog = 12; - params.searchLog = SEARCHLOG_MIN; - playAround(winners, params, srcBuffer, srcSize, ctx); - /* end summary */ - BMK_printWinners(winners, srcSize); + BMK_printWinners(f, winners, srcSize); /* clean up*/ + fclose(f); ZSTD_HC_freeCCtx(ctx); } @@ -612,7 +662,7 @@ int benchFiles(char** fileNamesTable, int nbFiles) return 11; } - // Memory allocation & restrictions + /* Memory allocation & restrictions */ inFileSize = BMK_GetFileSize(inFileName); benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3; if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; @@ -696,7 +746,7 @@ int main(int argc, char** argv) if(!argument) continue; // Protection if argument empty - // Decode command (note : aggregated commands are allowed) + /* Decode command (note : aggregated commands are allowed) */ if (argument[0]=='-') { while (argument[1]!=0)