#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
-#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */
+#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */
+
+static const char* g_stratName[ZSTD_btultra+1] = {
+ "(none) ", "ZSTD_fast ", "ZSTD_dfast ",
+ "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
+ "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
+
+
+static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
/*-************************************
* Setup for Adding new params
NUM_PARAMS = 8
} varInds_t;
+typedef struct {
+ U32 vals[NUM_PARAMS];
+} paramValues_t;
+
+/* list of parameters */
+static const varInds_t paramTable[NUM_PARAMS] =
+ { wlog_ind, clog_ind, hlog_ind, slog_ind, slen_ind, tlen_ind, strt_ind, fadt_ind };
+
/* maximum value of parameters */
static const U32 mintable[NUM_PARAMS] =
{ ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
static const U32 rangetable[NUM_PARAMS] =
{ WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
+/* ZSTD_cctxSetParameter() index to set */
static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] =
{ ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
+/* names of parameters */
static const char* g_paramNames[NUM_PARAMS] =
- { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"};
+ { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" };
+/* shortened names of parameters */
+static const char* g_shortParamNames[NUM_PARAMS] =
+ { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" };
-static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
-/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */
+/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
static U32 rangeMap(varInds_t param, U32 ind) {
ind = MIN(ind, rangetable[param] - 1);
switch(param) {
case strt_ind:
return mintable[param] + ind;
case NUM_PARAMS:
+ DISPLAY("Error, not a valid param\n ");
return (U32)-1;
}
return 0; /* should never happen, stop compiler warnings */
case strt_ind:
return value - mintable[param];
case NUM_PARAMS:
+ DISPLAY("Error, not a valid param\n ");
return (U32)-1;
}
return 0; /* should never happen, stop compiler warnings */
}
-typedef struct {
- U32 vals[NUM_PARAMS];
-} paramValues_t;
-
-static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
- ZSTD_compressionParameters c;
- memset(&c, 0, sizeof(ZSTD_compressionParameters));
- c.windowLog = p.vals[wlog_ind];
- c.chainLog = p.vals[clog_ind];
- c.hashLog = p.vals[hlog_ind];
- c.searchLog = p.vals[slog_ind];
- c.searchLength = p.vals[slen_ind];
- c.targetLength = p.vals[tlen_ind];
- c.strategy = p.vals[strt_ind];
- /* no forceAttachDict */
- return c;
-}
-
-/* 0 = auto for fadt */
-static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
- paramValues_t p;
- p.vals[wlog_ind] = c.windowLog;
- p.vals[clog_ind] = c.chainLog;
- p.vals[hlog_ind] = c.hashLog;
- p.vals[slog_ind] = c.searchLog;
- p.vals[slen_ind] = c.searchLength;
- p.vals[tlen_ind] = c.targetLength;
- p.vals[strt_ind] = c.strategy;
- p.vals[fadt_ind] = 0;
- return p;
-}
-
-/* equivalent of ZSTD_adjustCParams for paramValues_t */
-static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) {
- U32 fval = p.vals[fadt_ind];
- p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
- p.vals[fadt_ind] = fval;
- return p;
+/* display of params */
+static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
+ switch(param) {
+ case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break;
+ case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break;
+ case wlog_ind:
+ case clog_ind:
+ case hlog_ind:
+ case slog_ind:
+ case slen_ind:
+ case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break;
+ case NUM_PARAMS:
+ DISPLAY("Error, not a valid param\n "); break;
+ }
}
/*-************************************
static U32 g_noSeed = 0;
static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
-
+static U32 g_memoLimit = (U32)-1; //32 MB
typedef struct {
BMK_result_t result;
* Private functions
*********************************************************/
+static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
+ ZSTD_compressionParameters c;
+ memset(&c, 0, sizeof(ZSTD_compressionParameters));
+ c.windowLog = p.vals[wlog_ind];
+ c.chainLog = p.vals[clog_ind];
+ c.hashLog = p.vals[hlog_ind];
+ c.searchLog = p.vals[slog_ind];
+ c.searchLength = p.vals[slen_ind];
+ c.targetLength = p.vals[tlen_ind];
+ c.strategy = p.vals[strt_ind];
+ /* no forceAttachDict */
+ return c;
+}
+
+static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
+ paramValues_t p;
+ varInds_t i;
+ p.vals[wlog_ind] = c.windowLog;
+ p.vals[clog_ind] = c.chainLog;
+ p.vals[hlog_ind] = c.hashLog;
+ p.vals[slog_ind] = c.searchLog;
+ p.vals[slen_ind] = c.searchLength;
+ p.vals[tlen_ind] = c.targetLength;
+ p.vals[strt_ind] = c.strategy;
+
+ /* set all other params to their minimum value */
+ for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
+ p.vals[i] = mintable[i];
+ }
+ return p;
+}
+
+/* equivalent of ZSTD_adjustCParams for paramValues_t */
+static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) {
+ paramValues_t ot = p;
+ varInds_t i;
+ p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
+
+ /* retain value of all other parameters */
+ for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
+ p.vals[i] = ot.vals[i];
+ }
+ return p;
+}
+
/* accuracy in seconds only, span can be multiple years */
static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
return 0; \
} }
-
-/* Like ZSTD_checkCParams() but allows 0's */
-/* no check on targetLen? */
static int paramValid(paramValues_t paramTarget) {
U32 i;
for(i = 0; i < NUM_PARAMS; i++) {
U32 i;
fprintf(f,"--zstd=");
for(i = 0; i < NUM_PARAMS; i++) {
- fprintf(f,"%s", g_paramNames[i]);
- fprintf(f,"=%u", params.vals[i]);
+ fprintf(f,"%s=", g_paramNames[i]);
+
+ if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); }
+ else { displayParamVal(f, i, params.vals[i], 0); }
+
if(i != NUM_PARAMS - 1) {
fprintf(f, ",");
}
fprintf(f, "\n");
}
-
-static const char* g_stratName[ZSTD_btultra+1] = {
- "(none) ", "ZSTD_fast ", "ZSTD_dfast ",
- "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
- "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
-
static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) {
+ varInds_t v;
res.params = cParamUnsetMin(res.params);
- fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s}, ",
- res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind],
- res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]);
- fprintf(f,
- " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+ fprintf(f," {");
+ for(v = 0; v < NUM_PARAMS; v++) {
+ if(v != 0) { fprintf(f, ","); }
+ displayParamVal(f, v, res.params.vals[v], 3);
+ }
+
+ fprintf(f, "}, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
(double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
}
return p;
}
-static winnerInfo_t initWinnerInfo(paramValues_t p) {
+static winnerInfo_t initWinnerInfo(const paramValues_t p) {
winnerInfo_t w1;
w1.result.cSpeed = 0.;
w1.result.dSpeed = 0.;
void** resPtrs;
size_t* resSizes;
size_t nbBlocks;
+ size_t maxBlockSize;
} buffers_t;
typedef struct {
* From bench.c
*********************************************************/
-static void BMK_initCCtx(ZSTD_CCtx* ctx,
- const void* dictBuffer, const size_t dictBufferSize, const int cLevel,
- const paramValues_t* comprParams) {
- varInds_t i;
- ZSTD_CCtx_reset(ctx);
- ZSTD_CCtx_resetParameters(ctx);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
-
- for(i = 0; i < NUM_PARAMS; i++) {
- if(comprParams->vals[i] != PARAM_UNSET)
- ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]);
- }
- ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
-}
-
-static void BMK_initDCtx(ZSTD_DCtx* dctx,
- const void* dictBuffer, const size_t dictBufferSize) {
- ZSTD_DCtx_reset(dctx);
- ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
-}
-
typedef struct {
ZSTD_CCtx* ctx;
const void* dictBuffer;
static size_t local_initCCtx(void* payload) {
const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
- BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams);
+ varInds_t i;
+ ZSTD_CCtx_reset(ag->ctx);
+ ZSTD_CCtx_resetParameters(ag->ctx);
+ ZSTD_CCtx_setParameter(ag->ctx, ZSTD_p_compressionLevel, ag->cLevel);
+
+ for(i = 0; i < NUM_PARAMS; i++) {
+ if(ag->comprParams->vals[i] != PARAM_UNSET)
+ ZSTD_CCtx_setParameter(ag->ctx, cctxSetParamTable[i], ag->comprParams->vals[i]);
+ }
+ ZSTD_CCtx_loadDictionary(ag->ctx, ag->dictBuffer, ag->dictBufferSize);
+
return 0;
}
static size_t local_initDCtx(void* payload) {
const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload;
- BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
+ ZSTD_DCtx_reset(ag->dctx);
+ ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
return 0;
}
* From bench.c End
*********************************************************/
+static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
+ varInds_t v;
+ for(v = 0; v < NUM_PARAMS; v++) {
+ if(pc->vals[v] != PARAM_UNSET) {
+ U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]);
+ if(newval != pc->vals[v]) {
+ pc->vals[v] = newval;
+ DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n");
+ }
+ }
+ }
+
+ if(pc->vals[wlog_ind] != PARAM_UNSET) {
+
+ U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1;
+ /* edge case of highBit not working for 0 */
+
+ if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) {
+ U32 adjust = MAX(mintable[wlog_ind], sshb);
+ if(adjust != pc->vals[wlog_ind]) {
+ pc->vals[wlog_ind] = adjust;
+ DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]);
+ }
+ }
+ }
+
+ if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
+ U32 maxclog;
+ if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) {
+ maxclog = pc->vals[wlog_ind] + 1;
+ } else {
+ maxclog = pc->vals[wlog_ind];
+ }
+
+ if(pc->vals[clog_ind] > maxclog) {
+ pc->vals[clog_ind] = maxclog;
+ DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]);
+ }
+ }
+
+ if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) {
+ if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) {
+ pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1;
+ DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]);
+ }
+ }
+
+ if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
+ if(pc->vals[slog_ind] > pc->vals[clog_ind]) {
+ pc->vals[clog_ind] = pc->vals[slog_ind];
+ DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]);
+ }
+ }
+}
+
+/* what about low something like clog vs hlog in lvl 1? */
+static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t srcSize) {
+ return
+ (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
+ || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= srcSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
+ || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
+ || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
+ || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
+
+}
+
static void freeNonSrcBuffers(const buffers_t b) {
free(b.srcPtrs);
free(b.srcSizes);
buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
buff->dstSizes[0] = buff->dstCapacities[0];
buff->resSizes[0] = buff->srcSizes[0];
+ buff->maxBlockSize = buff->srcSizes[0];
for(n = 1; n < blockNb; n++) {
buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
buff->dstSizes[n] = buff->dstCapacities[n];
buff->resSizes[n] = buff->srcSizes[n];
+
+ buff->maxBlockSize = MAX(buff->maxBlockSize, buff->srcSizes[n]);
}
buff->nbBlocks = blockNb;
char* buffer = (char*)(srcBuffer);
size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
fclose(f);
- if (readSize != (size_t)fileSize) { /* should we accept partial read? */
+ if (readSize != (size_t)fileSize) {
DISPLAY("could not read %s", fileNamesTable[n]);
ret = 1;
goto _cleanUp;
}
static int BMK_benchParam(BMK_result_t* resultPtr,
- buffers_t buf, contexts_t ctx,
+ const buffers_t buf, const contexts_t ctx,
const paramValues_t cParams) {
BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
*resultPtr = res.result;
#define SPEED_RESULT 4
#define SIZE_RESULT 5
/* maybe have epsilon-eq to limit table size? */
-static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
+static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) {
if(r1.cSpeed < r2.cSpeed) {
if(r1.cSize >= r2.cSize) {
return BETTER_RESULT;
/* 0 for insertion, 1 for no insert */
/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
-static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
+static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) {
BMK_result_t r = w.result;
winner_ll_node* cur_node = g_winners;
/* first node to insert */
}
}
-static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
+static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize)
{
int cLevel;
}
-static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
+static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize)
{
fseek(f, 0, SEEK_SET);
BMK_printWinners2(f, winners, srcSize);
}
static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params,
- buffers_t buf, contexts_t ctx)
+ const buffers_t buf, const contexts_t ctx)
{
BMK_result_t testResult;
int better = 0;
size_t arrayLen = 1;
int i;
for(i = 0; i < varyLen; i++) {
- if(varyParams[i] != strt_ind) {
- arrayLen *= rangetable[varyParams[i]];
- }
+ if(varyParams[i] == strt_ind) continue; /* strategy separated by table */
+ arrayLen *= rangetable[varyParams[i]];
}
return arrayLen;
}
ind /= rangetable[v]; */
switch(varyParams[i]) {
- case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break;
- case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break;
- case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break;
- case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break;
- case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break;
- case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break;
- case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break;
+ case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break;
+ case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break;
+ case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break;
+ case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break;
+ case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break;
+ case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break;
+ case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break;
case strt_ind:
case NUM_PARAMS: break;
}
static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
size_t i;
size_t arrayLen = memoTableLen(varyParams, varyLen);
- int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET;
- int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET;
- int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET;
- int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET;
int j = 0;
assert(memoTable != NULL);
memset(memoTable, 0, arrayLen);
paramConstraints = cParamUnsetMin(paramConstraints);
-
for(i = 0; i < arrayLen; i++) {
memoTableIndInv(¶mConstraints, varyParams, varyLen, i);
- if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) {
- memoTable[i] = 255;
- j++;
- }
- if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) {
- memoTable[i] = 255;
- }
- /* nil out parameter sets equivalent to others. */
- if(cwFixed) {
- if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) {
- if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) {
- if(memoTable[i] != 255) { j++; }
- memoTable[i] = 255;
- }
- } else {
- if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) {
- if(memoTable[i] != 255) { j++; }
- memoTable[i] = 255;
- }
- }
- }
-
- if(scFixed) {
- if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) {
- if(memoTable[i] != 255) { j++; }
- memoTable[i] = 255;
- }
- }
-
- if(whFixed) {
- if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) {
- if(memoTable[i] != 255) { j++; }
- memoTable[i] = 255;
- }
+ if(redundantParams(paramConstraints, target, srcSize)) {
+ memoTable[i] = 255; j++;
}
}
- DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i);
- if((int)i == j) {
- DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]);
- }
+
+ DEBUGOUTPUT("%d / %d Invalid\n", j, (int)arrayLen);
}
/* frees all allocated memotables */
/* inits memotables for all (including mallocs), all strategies */
/* takes unsanitized varyParams */
-static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
+static U8** createMemoTableArray(paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
varInds_t varNew[NUM_PARAMS];
U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1));
int i;
for(i = 1; i <= (int)ZSTD_btultra; i++) {
const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i);
- mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
+ size_t mtl = memoTableLen(varNew, varLenNew);
+ if(mtl > g_memoLimit) { mtAll[i] = NULL; continue; }
+ mtAll[i] = malloc(sizeof(U8) * mtl);
if(mtAll[i] == NULL) {
freeMemoTableArray(mtAll);
return NULL;
}
+ paramConstraints.vals[strt_ind] = i;
initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize);
}
return mtAll;
}
-static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) {
+static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) {
U32 i;
for(i = 0; i < NUM_PARAMS; i++) {
if(mask.vals[i] != PARAM_UNSET) {
static void playAround(FILE* f, oldWinnerInfo_t* winners,
ZSTD_compressionParameters params,
- buffers_t buf, contexts_t ctx)
+ const buffers_t buf, const contexts_t ctx)
{
int nbVariations = 0;
UTIL_time_t const clockStart = UTIL_getTime();
}
-/* Completely random parameter selection */
-static ZSTD_compressionParameters randomParams(void)
+/* Sets pc to random unmeasured set of parameters */
+/* Doesn't do strategy! */
+static void randomConstrainedParams(paramValues_t* pc, const varInds_t* varArray, const int varLen, const U8* memoTable)
{
- ZSTD_compressionParameters p;
- U32 validated = 0;
- while (!validated) {
- /* totally random entry */
- p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN))
- + ZSTD_CHAINLOG_MIN;
- p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN))
- + ZSTD_HASHLOG_MIN;
- p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN))
- + ZSTD_SEARCHLOG_MIN;
- p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN))
- + ZSTD_WINDOWLOG_MIN;
- p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN))
- + ZSTD_SEARCHLENGTH_MIN;
- p.targetLength=(FUZ_rand(&g_rand) % (512));
-
- p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1));
-
- validated = !ZSTD_isError(ZSTD_checkCParams(p));
+ size_t j;
+ for(j = 0; j < MIN(g_memoLimit, memoTableLen(varArray, varLen)); j++) {
+ int i;
+ for(i = 0; i < NUM_PARAMS; i++) {
+ varInds_t v = varArray[i];
+ if(v == strt_ind) continue; //don't do strategy (dependent on MT)
+ pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
+ }
+
+ if(memoTable == NULL || memoTable[memoTableInd(pc, varArray, varLen)]) { break; }
}
- return p;
}
-/* Sets pc to random unmeasured set of parameters */
-static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable)
+/* Completely random parameter selection */
+static ZSTD_compressionParameters randomParams(void)
{
- size_t tries = memoTableLen(varArray, varLen);
- const size_t maxSize = memoTableLen(varArray, varLen);
- size_t ind;
- do {
- ind = (FUZ_rand(&g_rand)) % maxSize;
- tries--;
- } while(memoTable[ind] > 0 && tries > 0);
-
- memoTableIndInv(pc, varArray, varLen, (unsigned)ind);
+ paramValues_t p;
+ p.vals[strt_ind] = rangeMap(strt_ind, rangeMap(strt_ind, FUZ_rand(&g_rand) % rangetable[strt_ind]));
+ randomConstrainedParams(&p, paramTable, NUM_PARAMS, NULL);
+ return pvalsToCParams(p);
}
static void BMK_selectRandomStart(
FILE* f, oldWinnerInfo_t* winners,
- buffers_t buf, contexts_t ctx)
+ const buffers_t buf, const contexts_t ctx)
{
U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
if ((id==0) || (winners[id].params.windowLog==0)) {
}
}
-static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize)
+static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
{
ZSTD_compressionParameters params;
oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1];
BMK_init_level_constraints(g_target * (1 MB));
} else {
/* baseline config for level 1 */
- ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize);
+ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize);
BMK_result_t testResult;
BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params));
BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
int i;
for (i=0; i<=maxSeeds; i++) {
- params = ZSTD_getCParams(i, maxBlockSize, 0);
+ params = ZSTD_getCParams(i, buf.maxBlockSize, 0);
BMK_seed(winners, params, buf, ctx);
} }
BMK_printWinners(f, winners, buf.srcSize);
fclose(f);
}
-static int benchOnce(buffers_t buf, contexts_t ctx) {
+static int benchOnce(const buffers_t buf, const contexts_t ctx) {
BMK_result_t testResult;
if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
{
const char* const name = "Sample 10MB";
size_t const benchedSize = 10 MB;
- U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
void* srcBuffer = malloc(benchedSize);
int ret = 0;
if(g_singleRun) {
ret = benchOnce(buf, ctx);
} else {
- BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
+ BMK_benchFullTable(buf, ctx);
}
freeBuffers(buf);
{
buffers_t buf;
contexts_t ctx;
- size_t maxBlockSize = 0, i;
int ret = 0;
if(createBuffers(&buf, fileNamesTable, nbFiles)) {
return 2;
}
- for(i = 0; i < buf.nbBlocks; i++) {
- maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]);
- }
-
DISPLAY("\r%79s\r", "");
if(nbFiles == 1) {
DISPLAY("using %s : \n", fileNamesTable[0]);
DISPLAY("using %d Files : \n", nbFiles);
}
- g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
+ g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
if(g_singleRun) {
ret = benchOnce(buf, ctx);
} else {
- BMK_benchFullTable(buf, ctx, maxBlockSize);
+ BMK_benchFullTable(buf, ctx);
}
freeBuffers(buf);
size_t memind = memoTableInd(&cParams, varyParams, varyLen);
int res;
- if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; }
+ if((memoTable == NULL && redundantParams(cParams, target, buf.maxBlockSize)) || ((memoTable != NULL) && memoTable[memind] >= INFEASIBLE_THRESHOLD)) { return WORSE_RESULT; }
res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
}
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize);
- if(res == BETTER_RESULT || feas) {
+ if(memoTable != NULL && (res == BETTER_RESULT || feas)) {
memoTable[memind] = 255;
}
return res;
static winnerInfo_t climbOnce(const constraint_t target,
const varInds_t* varArray, const int varLen, ZSTD_strategy strat,
U8** memoTableArray,
- buffers_t buf, contexts_t ctx,
+ const buffers_t buf, const contexts_t ctx,
const paramValues_t init) {
/*
* cparam - currently considered 'center'
/* so climb is given the right fixed strategy */
paramTarget.vals[strt_ind] = strat;
/* to pass ZSTD_checkCParams */
-
paramTarget = cParamUnsetMin(paramTarget);
init = paramTarget;
const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
winnerInfo_t winner = initWinnerInfo(emptyParams());
U8** allMT = NULL;
- paramValues_t paramBase = cParamUnsetMin(paramTarget);
- size_t k, maxBlockSize = 0;
+ paramValues_t paramBase;
contexts_t ctx;
buffers_t buf;
g_time = UTIL_getTime();
- if(!paramValid(paramBase)) {
- return 1;
- }
-
if(createBuffers(&buf, fileNamesTable, nbFiles)) {
DISPLAY("unable to load files\n");
return 1;
DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles);
}
-
- for(k = 0; k < buf.nbBlocks; k++) {
- maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize);
- }
+ /* sanitize paramTarget */
+ optimizerAdjustInput(¶mTarget, buf.maxBlockSize);
+ paramBase = cParamUnsetMin(paramTarget);
/* if strategy is fixed, only init that part of memotable */
if(paramTarget.vals[strt_ind] != PARAM_UNSET) {
goto _cleanUp;
}
- initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize);
+ initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, buf.maxBlockSize);
} else {
- allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize);
+ allMT = createMemoTableArray(paramTarget, target, varArray, varLen, buf.maxBlockSize);
}
if(!allMT) {
/* use level'ing mode instead of normal target mode */
/* Should lvl be parameter-masked here? */
if(g_optmode) {
- winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize));
+ winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize));
if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
ret = 3;
goto _cleanUp;
/* Don't want it to return anything worse than the best known result */
if(g_singleRun) {
BMK_result_t res;
- g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
+ g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
if(BMK_benchParam(&res, buf, ctx, g_params)) {
ret = 45;
goto _cleanUp;
int i;
for (i=1; i<=maxSeeds; i++) {
int ec;
- CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget);
+ CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, ctx.dictSize)), paramTarget);
ec = BMK_benchParam(&candidate, buf, ctx, CParams);
BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
}
#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
-#define PARSE_CPARAMS(variable) \
-{ \
- PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \
- PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \
- PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \
- PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \
- PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \
- PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \
- PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \
- PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[fadt_ind]); \
+/* 1 if successful parse, 0 otherwise */
+static int parse_params(const char** argptr, paramValues_t* pv) {
+ int matched = 0;
+ const char* argOrig = *argptr;
+ varInds_t v;
+ for(v = 0; v < NUM_PARAMS; v++) {
+ if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) {
+ if(**argptr == '=') {
+ (*argptr)++;
+ pv->vals[v] = readU32FromChar(argptr);
+ matched = 1;
+ break;
+ }
+ }
+ /* reset and try again */
+ *argptr = argOrig;
+ }
+ return matched;
}
int main(int argc, const char** argv)
if (longCommandWArg(&argument, "--optimize=")) {
g_optimizer = 1;
for ( ; ;) {
- PARSE_CPARAMS(paramTarget);
+ if(parse_params(&argument, ¶mTarget)) { if(argument[0] == ',') { argument++; continue; } else break; }
PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed);
PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed);
PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
+ PARSE_SUB_ARGS("memoLimit=", "memo=", g_memoLimit);
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid optimization parameter \n");
/* Decode command (note : aggregated commands are allowed) */
g_singleRun = 1;
for ( ; ;) {
- PARSE_CPARAMS(g_params)
+ if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid compression parameter \n");
projects / thirdparty / zstd.git / commitdiff
