From: George Lu Date: Tue, 31 Jul 2018 00:42:46 +0000 (-0700) Subject: Update fulltable to use same interface X-Git-Tag: v0.0.29~43^2~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d4730a4f66746e6fe1473f06eb2d9bb49c831b83;p=thirdparty%2Fzstd.git Update fulltable to use same interface Add seperateFiles flag --- diff --git a/tests/README.md b/tests/README.md index 946f890b1..736916936 100644 --- a/tests/README.md +++ b/tests/README.md @@ -125,6 +125,7 @@ Full list of arguments -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output -D : Next argument dictionary file + -s : Benchmark all files separately ``` Any inputs afterwards are treated as files to benchmark. diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 7e86c2e9e..cba97ebbb 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -96,7 +96,7 @@ typedef enum { static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ -* Benchmark Parameters +* Benchmark Parameters/Global Variables **************************************/ typedef BYTE U8; @@ -110,7 +110,7 @@ static U32 g_singleRun = 0; static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; +static ZSTD_compressionParameters g_params; static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ @@ -150,6 +150,14 @@ void BMK_SetNbIterations(int nbLoops) DISPLAY("- %u iterations -\n", g_nbIterations); } +/* + * Additional Global Variables (Defined Above Use) + * g_stratName + * g_level_constraint + * g_alreadyTested + * g_maxTries + */ + /*-******************************************************* * Private functions *********************************************************/ @@ -335,17 +343,6 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* benchParam but only takes in one input buffer. */ -static int -BMK_benchParam1(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - const ZSTD_compressionParameters cParams) { - - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); - *resultPtr = res.result; - return res.error; -} - static ZSTD_compressionParameters emptyParams(void) { ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; return p; @@ -381,14 +378,6 @@ typedef struct { ZSTD_DCtx* dctx; } contexts_t; -static int BMK_benchParam(BMK_result_t* resultPtr, - const buffers_t buf, const contexts_t ctx, - const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files"); - *resultPtr = res.result; - return res.error; -} - /*-******************************************************* * From Paramgrill *********************************************************/ @@ -542,7 +531,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab size_t n; U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); - const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize) + const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; @@ -830,6 +819,14 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t return results; } +static int BMK_benchParam(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, + const ZSTD_compressionParameters cParams) { + BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3); + *resultPtr = res.result; + return res.error; +} + /* comparison function: */ /* strictly better, strictly worse, equal, speed-side adv, size-side adv */ //Maybe use compress_only for benchmark first run? @@ -911,7 +908,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { } - //assert(cur_node->next == NULL) + assert(cur_node->next == NULL); switch(speedSizeCompare(r, cur_node->res.result)) { case BETTER_RESULT: { @@ -995,7 +992,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } - //prints out tradeoff table if using lvl + //prints out tradeoff table if using lvloptimize if(g_optmode && g_optimizer) { winnerInfo_t w; winner_ll_node* n; @@ -1042,12 +1039,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); } - -#if 0 - if(BMK_timeSpan(g_time) > g_grillDuration_s) { - exit(0); - } -#endif } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -1099,14 +1090,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } } } -static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize) +static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, + buffers_t buf, contexts_t ctx) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam1(&testResult, srcBuffer, srcSize, params); + BMK_benchParam(&testResult, buf, ctx, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1122,15 +1113,15 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* first solution for this cLevel */ winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; continue; } if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) { /* Validate solution is "good enough" */ - double W_ratio = (double)srcSize / testResult.cSize; - double O_ratio = (double)srcSize / winners[cLevel].result.cSize; + double W_ratio = (double)buf.srcSize / testResult.cSize; + double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize; double W_ratioNote = log (W_ratio); double O_ratioNote = log (O_ratio); size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); @@ -1187,7 +1178,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; } } @@ -1315,6 +1306,26 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var *ptr = p; } +/* maybe put strategy back in */ +static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) +{ + ZSTD_compressionParameters p; + U32 validated = 0; + while (!validated) { + U32 i; + p = *ptr; + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1); + if(changeID < (U32)(varyLen << 1)) { + paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); + } else { + p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */ + } + } + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + } + *ptr = p; +} /* length of memo table given free variables */ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; @@ -1463,6 +1474,17 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co return mtAll; } +static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { + base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; + base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; + base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; + base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; + base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; + base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; + base.strategy = mask.strategy ? mask.strategy : base.strategy; + return base; +} + #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1< g_maxNbVariations) break; - paramVariation(&p, unconstrained, 7, 4); + paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) @@ -1500,11 +1522,11 @@ static void playAround(FILE* f, winnerInfo_t* winners, /* test */ b = NB_TESTS_PLAYED(p); (*b)++; - if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue; + if (!BMK_seed(winners, p, buf, ctx)) continue; /* improvement found => search more */ - BMK_printWinners(f, winners, srcSize); - playAround(f, winners, p, srcBuffer, srcSize); + BMK_printWinners(f, winners, buf.srcSize); + playAround(f, winners, p, buf, ctx); } } @@ -1551,35 +1573,24 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, - const void* srcBuffer, size_t srcSize) + buffers_t buf, contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { /* use some random entry */ - ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); - playAround(f, winners, p, srcBuffer, srcSize); + ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0); + playAround(f, winners, p, buf, ctx); } else { - playAround(f, winners, winners[id].params, srcBuffer, srcSize); + playAround(f, winners, winners[id].params, buf, ctx); } } -static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) -{ - BMK_result_t testResult; - g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); - DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); - return; -} - -static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) +static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) { ZSTD_compressionParameters params; winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1590,9 +1601,9 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); + ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? BMK_result_t testResult; - BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params); + BMK_benchParam(&testResult, buf, ctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1600,157 +1611,158 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=0; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, blockSize, 0); - BMK_seed(winners, params, srcBuffer, srcSize); + params = ZSTD_getCParams(i, maxBlockSize, 0); + BMK_seed(winners, params, buf, ctx); } } - BMK_printWinners(f, winners, srcSize); + BMK_printWinners(f, winners, buf.srcSize); /* start tests */ { const time_t grillStart = time(NULL); do { - BMK_selectRandomStart(f, winners, srcBuffer, srcSize); + BMK_selectRandomStart(f, winners, buf, ctx); } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } /* end summary */ - BMK_printWinners(f, winners, srcSize); + BMK_printWinners(f, winners, buf.srcSize); DISPLAY("grillParams operations completed \n"); /* clean up*/ fclose(f); } -static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) -{ - if (g_singleRun) - return BMK_benchOnce(srcBuffer, srcSize); - else - return BMK_benchFullTable(srcBuffer, srcSize); -} - - static int benchSample(void) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; + U32 blockSize = g_blockSize ? g_blockSize : benchedSize; + U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; + size_t splitSize = 0; - void* origBuff = malloc(benchedSize); - if (!origBuff) { perror("not enough memory"); return 12; } + buffers_t buf; + contexts_t ctx; - /* Fill buffer */ - RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); + buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); + buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.srcSize = benchedSize; + + if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) { + DISPLAY("Allocation Error\n"); + freeBuffers(buf); + return 1; + } - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - BMK_benchMemInit(origBuff, benchedSize); + buf.srcBuffer = malloc(benchedSize); + buf.srcPtrs[0] = (const void*)buf.srcBuffer; + buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks); + buf.resPtrs[0] = malloc(benchedSize); - free(origBuff); - return 0; -} + if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) { + DISPLAY("Allocation Error\n"); + freeBuffers(buf); + return 1; + } -static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) { - buffers_t buf; - contexts_t ctx; - BMK_result_t testResult; - size_t maxBlockSize = 0, i; + splitSize = MIN(benchedSize, blockSize); + buf.srcSizes[0] = splitSize; + buf.dstCapacities[0] = ZSTD_compressBound(splitSize); + buf.resSizes[0] = splitSize; - if(createBuffers(&buf, fileNamesTable, nbFiles)) { - DISPLAY("unable to load files\n"); - return 1; + for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) { + const size_t i = buf.nbBlocks; + const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize); + buf.srcSizes[i] = nextBlockSize; + buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize); + buf.resSizes[i] = nextBlockSize; + buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]); + buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]); + buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]); + splitSize += nextBlockSize; } - if(createContexts(&ctx, dictFileName)) { - DISPLAY("unable to load dictionary\n"); + if(createContexts(&ctx, NULL)) { + DISPLAY("Context Creation Error\n"); freeBuffers(buf); - return 2; + return 1; } - for(i = 0; i < buf.nbBlocks; i++) { - maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]); - } + RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0); + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); + + BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); + + freeBuffers(buf); + freeContexts(ctx); + + return 0; +} - g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0); + +static int benchOnce(buffers_t buf, contexts_t ctx) { + BMK_result_t testResult; if(BMK_benchParam(&testResult, buf, ctx, g_params)) { DISPLAY("Error during benchmarking\n"); - freeBuffers(buf); - freeContexts(ctx); - return 3; + return 1; } DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); - - freeBuffers(buf); - freeContexts(ctx); return 0; } /* benchFiles() : * note: while this function takes a table of filenames, * in practice, only the first filename will be used */ -//TODO: dictionaries still not supported in fullTable mode -int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName) +int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel) { - int fileIdx=0; + buffers_t buf; + contexts_t ctx; + size_t maxBlockSize = 0, i; + int ret = 0; - if(g_singleRun) { - return benchOnce(fileNamesTable, nbFiles, dictFileName); + if(createBuffers(&buf, fileNamesTable, nbFiles)) { + DISPLAY("unable to load files\n"); + return 1; } - /* Loop for each file */ - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - origBuff = malloc(benchedSize); - if (origBuff==NULL) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } + for(i = 0; i < buf.nbBlocks; i++) { + maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]); + } - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } + DISPLAY("\r%79s\r", ""); + if(nbFiles == 1) { + DISPLAY("using %s : \n", fileNamesTable[0]); + } else { + DISPLAY("using %d Files : \n", nbFiles); + } - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("using %s : \n", inFileName); - BMK_benchMemInit(origBuff, benchedSize); + g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); - /* clean */ - free(origBuff); + if(g_singleRun) { + ret = benchOnce(buf, ctx); + } else { + BMK_benchFullTable(buf, ctx, maxBlockSize); } - return 0; + freeBuffers(buf); + freeContexts(ctx); + return ret; } /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ @@ -2074,17 +2086,6 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } } -static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { - base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; - base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; - base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; - base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; - base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; - base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; - base.strategy = mask.strategy ? mask.strategy : base.strategy; - return base; -} - /* experiment with playing with this and decay value */ /* main fn called when using --optimize */ @@ -2115,6 +2116,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ contexts_t ctx; buffers_t buf; + g_time = UTIL_getTime(); + /* Init */ if(!cParamValid(paramTarget)) { return 1; @@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ g_lvltarget.cSize /= ((double)g_strictness) / 100; target.cSpeed = (U32)g_lvltarget.cSpeed; - target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth + target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2214,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } else { DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } @@ -2369,6 +2373,7 @@ static int usage_advanced(void) DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); DISPLAY( " -D : Next argument dictionary file\n"); + DISPLAY( " -s : Seperate Files\n"); return 0; } @@ -2401,13 +2406,13 @@ int main(int argc, const char** argv) const char* input_filename = NULL; const char* dictFileName = NULL; U32 main_pause = 0; - int optimizerCLevel = 0; + int cLevel = 0; + int seperateFiles = 0; constraint_t target = { 0, 0, (U32)-1 }; - ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; - - g_time = UTIL_getTime(); + ZSTD_compressionParameters paramTarget = emptyParams(); + g_params = emptyParams(); assert(argc>=1); /* for exename */ @@ -2430,11 +2435,11 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); - PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); + PARSE_SUB_ARGS("level=", "lvl=", cLevel); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); - PARSE_SUB_ARGS("maxTries", "tries", g_maxTries); + PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); DISPLAY("invalid optimization parameter \n"); return 1; @@ -2448,10 +2453,11 @@ int main(int argc, const char** argv) } else if (longCommandWArg(&argument, "--zstd=")) { /* Decode command (note : aggregated commands are allowed) */ g_singleRun = 1; - g_params = ZSTD_getCParams(2, g_blockSize, 0); + cLevel = 2; for ( ; ;) { PARSE_CPARAMS(g_params) - if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } + DISPLAY("invalid compression parameter \n"); return 1; } @@ -2528,8 +2534,8 @@ int main(int argc, const char** argv) continue; case 'L': { argument++; - int const cLevel = readU32FromChar(&argument); - g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); + cLevel = readU32FromChar(&argument); + g_params = emptyParams(); continue; } default : ; @@ -2558,6 +2564,10 @@ int main(int argc, const char** argv) g_grillDuration_s = (double)readU32FromChar(&argument); break; + case 's': + seperateFiles = 1; + break; + /* load dictionary file (only applicable for optimizer rn) */ case 'D': if(i == argc - 1) { /* last argument, return error. */ @@ -2588,11 +2598,24 @@ int main(int argc, const char** argv) result = benchSample(); } } else { - if (g_optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); + if(seperateFiles) { + for(i = 0; i < argc - filenamesStart; i++) { + if (g_optimizer) { + result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevel); + if(result) { DISPLAY("Error on File %d", i); return result; } + } else { + result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevel); + if(result) { DISPLAY("Error on File %d", i); return result; } + } + } } else { - result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName); - } } + if (g_optimizer) { + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevel); + } else { + result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevel); + } + } + } if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }