/*-*************************************
* Console display
+*
+* Captures the `displayLevel` variable in the local scope.
***************************************/
-#ifndef LOCALDISPLAYLEVEL
-static int g_displayLevel = 0;
-#endif
#undef DISPLAY
#define DISPLAY(...) \
{ \
fprintf(stderr, __VA_ARGS__); \
fflush(stderr); \
}
-#undef LOCALDISPLAYLEVEL
-#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
+#undef DISPLAYLEVEL
+#define DISPLAYLEVEL(l, ...) \
if (displayLevel >= l) { \
DISPLAY(__VA_ARGS__); \
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
-#undef DISPLAYLEVEL
-#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
-#ifndef LOCALDISPLAYUPDATE
-static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
-static clock_t g_time = 0;
-#endif
-#undef LOCALDISPLAYUPDATE
-#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
+#undef DISPLAYUPDATE
+#define DISPLAYUPDATE(lastUpdateTime, l, ...) \
if (displayLevel >= l) { \
- if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
- g_time = clock(); \
+ const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; \
+ if ((clock() - lastUpdateTime > refreshRate) || (displayLevel >= 4)) { \
+ lastUpdateTime = clock(); \
DISPLAY(__VA_ARGS__); \
} \
}
-#undef DISPLAYUPDATE
-#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
/*-*************************************
* Hash table
U32 *freqs;
U32 *dmerAt;
unsigned d;
+ int displayLevel;
} COVER_ctx_t;
#if defined(ZSTD_USE_C90_QSORT) \
*/
static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
const size_t *samplesSizes, unsigned nbSamples,
- unsigned d, double splitPoint)
+ unsigned d, double splitPoint, int displayLevel)
{
const BYTE *const samples = (const BYTE *)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
+ ctx->displayLevel = displayLevel;
/* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
if (ratio >= 10) {
return;
}
- LOCALDISPLAYLEVEL(displayLevel, 1,
- "WARNING: The maximum dictionary size %u is too large "
- "compared to the source size %u! "
- "size(source)/size(dictionary) = %f, but it should be >= "
- "10! This may lead to a subpar dictionary! We recommend "
- "training on sources at least 10x, and preferably 100x "
- "the size of the dictionary! \n", (U32)maxDictSize,
- (U32)nbDmers, ratio);
+ DISPLAYLEVEL(1,
+ "WARNING: The maximum dictionary size %u is too large "
+ "compared to the source size %u! "
+ "size(source)/size(dictionary) = %f, but it should be >= "
+ "10! This may lead to a subpar dictionary! We recommend "
+ "training on sources at least 10x, and preferably 100x "
+ "the size of the dictionary! \n", (U32)maxDictSize,
+ (U32)nbDmers, ratio);
}
COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize,
const size_t maxZeroScoreRun = MAX(10, MIN(100, epochs.num >> 3));
size_t zeroScoreRun = 0;
size_t epoch;
+ clock_t lastUpdateTime = 0;
+ const int displayLevel = ctx->displayLevel;
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
(U32)epochs.num, (U32)epochs.size);
/* Loop through the epochs until there are no more segments or the dictionary
tail -= segmentSize;
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
DISPLAYUPDATE(
+ lastUpdateTime,
2, "\r%u%% ",
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
}
BYTE* const dict = (BYTE*)dictBuffer;
COVER_ctx_t ctx;
COVER_map_t activeDmers;
+ const int displayLevel = parameters.zParams.notificationLevel;
parameters.splitPoint = 1.0;
- /* Initialize global data */
- g_displayLevel = (int)parameters.zParams.notificationLevel;
/* Checks */
if (!COVER_checkParameters(parameters, dictBufferCapacity)) {
DISPLAYLEVEL(1, "Cover parameters incorrect\n");
/* Initialize context and activeDmers */
{
size_t const initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
- parameters.d, parameters.splitPoint);
+ parameters.d, parameters.splitPoint, displayLevel);
if (ZSTD_isError(initVal)) {
return initVal;
}
}
- COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, g_displayLevel);
+ COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, displayLevel);
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
COVER_ctx_destroy(&ctx);
BYTE* const dict = (BYTE*)malloc(dictBufferCapacity);
COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
U32* const freqs = (U32*)malloc(ctx->suffixSize * sizeof(U32));
+ const int displayLevel = ctx->displayLevel;
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
goto _cleanup;
(1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
const unsigned shrinkDict = 0;
/* Local variables */
- const int displayLevel = (int)parameters->zParams.notificationLevel;
+ int displayLevel = (int)parameters->zParams.notificationLevel;
unsigned iteration = 1;
unsigned d;
unsigned k;
COVER_best_t best;
POOL_ctx *pool = NULL;
int warned = 0;
+ clock_t lastUpdateTime = 0;
/* Checks */
if (splitPoint <= 0 || splitPoint > 1) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
+ DISPLAYLEVEL(1, "Incorrect parameters\n");
return ERROR(parameter_outOfBound);
}
if (kMinK < kMaxD || kMaxK < kMinK) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
+ DISPLAYLEVEL(1, "Incorrect parameters\n");
return ERROR(parameter_outOfBound);
}
if (nbSamples == 0) {
}
/* Initialization */
COVER_best_init(&best);
- /* Turn down global display level to clean up display at level 2 and below */
- g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
/* Loop through d first because each new value needs a new context */
- LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
+ DISPLAYLEVEL(2, "Trying %u different sets of parameters\n",
kIterations);
for (d = kMinD; d <= kMaxD; d += 2) {
/* Initialize the context for this value of d */
COVER_ctx_t ctx;
- LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
+ DISPLAYLEVEL(3, "d=%u\n", d);
{
- const size_t initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint);
+ /* Turn down global display level to clean up display at level 2 and below */
+ const int childDisplayLevel = (displayLevel == 0) ? 0 : displayLevel - 1;
+ const size_t initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, childDisplayLevel);
if (ZSTD_isError(initVal)) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
+ DISPLAYLEVEL(1, "Failed to initialize context\n");
COVER_best_destroy(&best);
POOL_free(pool);
return initVal;
/* Prepare the arguments */
COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc(
sizeof(COVER_tryParameters_data_t));
- LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
+ DISPLAYLEVEL(3, "k=%u\n", k);
if (!data) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
+ DISPLAYLEVEL(1, "Failed to allocate parameters\n");
COVER_best_destroy(&best);
COVER_ctx_destroy(&ctx);
POOL_free(pool);
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict;
- data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel;
+ data->parameters.zParams.notificationLevel = (unsigned)ctx.displayLevel;
/* Check the parameters */
if (!COVER_checkParameters(data->parameters, dictBufferCapacity)) {
DISPLAYLEVEL(1, "Cover parameters incorrect\n");
COVER_tryParameters(data);
}
/* Print status */
- LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
- (unsigned)((iteration * 100) / kIterations));
+ DISPLAYUPDATE(lastUpdateTime, 2, "\r%u%% ",
+ (unsigned)((iteration * 100) / kIterations));
++iteration;
}
COVER_best_wait(&best);
COVER_ctx_destroy(&ctx);
}
- LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "\r%79s\r", "");
/* Fill the output buffer and parameters with output of the best parameters */
{
const size_t dictSize = best.dictSize;
/*-*************************************
* Console display
+*
+* Captures the `displayLevel` variable in the local scope.
***************************************/
-#ifndef LOCALDISPLAYLEVEL
-static int g_displayLevel = 0;
-#endif
#undef DISPLAY
#define DISPLAY(...) \
{ \
fprintf(stderr, __VA_ARGS__); \
fflush(stderr); \
}
-#undef LOCALDISPLAYLEVEL
-#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
+#undef DISPLAYLEVEL
+#define DISPLAYLEVEL(l, ...) \
if (displayLevel >= l) { \
DISPLAY(__VA_ARGS__); \
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
-#undef DISPLAYLEVEL
-#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
-#ifndef LOCALDISPLAYUPDATE
-static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
-static clock_t g_time = 0;
-#endif
-#undef LOCALDISPLAYUPDATE
-#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
+#undef DISPLAYUPDATE
+#define DISPLAYUPDATE(lastUpdateTime, l, ...) \
if (displayLevel >= l) { \
- if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
- g_time = clock(); \
+ const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; \
+ if ((clock() - lastUpdateTime > refreshRate) || (displayLevel >= 4)) { \
+ lastUpdateTime = clock(); \
DISPLAY(__VA_ARGS__); \
} \
}
-#undef DISPLAYUPDATE
-#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
/*-*************************************
unsigned d;
unsigned f;
FASTCOVER_accel_t accelParams;
+ int displayLevel;
} FASTCOVER_ctx_t;
const void* samplesBuffer,
const size_t* samplesSizes, unsigned nbSamples,
unsigned d, double splitPoint, unsigned f,
- FASTCOVER_accel_t accelParams)
+ FASTCOVER_accel_t accelParams,
+ int displayLevel)
{
const BYTE* const samples = (const BYTE*)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
+ ctx->displayLevel = displayLevel;
/* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
const COVER_epoch_info_t epochs = COVER_computeEpochs(
(U32)dictBufferCapacity, (U32)ctx->nbDmers, parameters.k, 1);
const size_t maxZeroScoreRun = 10;
+ const int displayLevel = ctx->displayLevel;
size_t zeroScoreRun = 0;
+ clock_t lastUpdateTime = 0;
size_t epoch;
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
(U32)epochs.num, (U32)epochs.size);
tail -= segmentSize;
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
DISPLAYUPDATE(
+ lastUpdateTime,
2, "\r%u%% ",
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
}
BYTE *const dict = (BYTE*)malloc(dictBufferCapacity);
COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
U32* freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
+ const int displayLevel = ctx->displayLevel;
if (!segmentFreqs || !dict || !freqs) {
DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
goto _cleanup;
FASTCOVER_ctx_t ctx;
ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams;
- /* Initialize global data */
- g_displayLevel = (int)parameters.zParams.notificationLevel;
+ const int displayLevel = (int)parameters.zParams.notificationLevel;
/* Assign splitPoint and f if not provided */
parameters.splitPoint = 1.0;
parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f;
{
size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
coverParams.d, parameters.splitPoint, parameters.f,
- accelParams);
+ accelParams, displayLevel);
if (ZSTD_isError(initVal)) {
DISPLAYLEVEL(1, "Failed to initialize context\n");
return initVal;
}
}
- COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel);
+ COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
/* Build the dictionary */
DISPLAYLEVEL(2, "Building dictionary\n");
{
COVER_best_t best;
POOL_ctx *pool = NULL;
int warned = 0;
+ clock_t lastUpdateTime = 0;
/* Checks */
if (splitPoint <= 0 || splitPoint > 1) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
+ DISPLAYLEVEL(1, "Incorrect splitPoint\n");
return ERROR(parameter_outOfBound);
}
if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect accel\n");
+ DISPLAYLEVEL(1, "Incorrect accel\n");
return ERROR(parameter_outOfBound);
}
if (kMinK < kMaxD || kMaxK < kMinK) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n");
+ DISPLAYLEVEL(1, "Incorrect k\n");
return ERROR(parameter_outOfBound);
}
if (nbSamples == 0) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "FASTCOVER must have at least one input file\n");
+ DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "dictBufferCapacity must be at least %u\n",
+ DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
ZDICT_DICTSIZE_MIN);
return ERROR(dstSize_tooSmall);
}
memset(&coverParams, 0 , sizeof(coverParams));
FASTCOVER_convertToCoverParams(*parameters, &coverParams);
accelParams = FASTCOVER_defaultAccelParameters[accel];
- /* Turn down global display level to clean up display at level 2 and below */
- g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
/* Loop through d first because each new value needs a new context */
- LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
- kIterations);
+ DISPLAYLEVEL(2, "Trying %u different sets of parameters\n", kIterations);
for (d = kMinD; d <= kMaxD; d += 2) {
/* Initialize the context for this value of d */
FASTCOVER_ctx_t ctx;
- LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
+ DISPLAYLEVEL(3, "d=%u\n", d);
{
- size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams);
+ /* Turn down global display level to clean up display at level 2 and below */
+ const int childDisplayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
+ size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams, childDisplayLevel);
if (ZSTD_isError(initVal)) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
+ DISPLAYLEVEL(1, "Failed to initialize context\n");
COVER_best_destroy(&best);
POOL_free(pool);
return initVal;
/* Prepare the arguments */
FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc(
sizeof(FASTCOVER_tryParameters_data_t));
- LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
+ DISPLAYLEVEL(3, "k=%u\n", k);
if (!data) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
+ DISPLAYLEVEL(1, "Failed to allocate parameters\n");
COVER_best_destroy(&best);
FASTCOVER_ctx_destroy(&ctx);
POOL_free(pool);
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict;
- data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel;
+ data->parameters.zParams.notificationLevel = (unsigned)ctx.displayLevel;
/* Check the parameters */
if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity,
data->ctx->f, accel)) {
FASTCOVER_tryParameters(data);
}
/* Print status */
- LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
- (unsigned)((iteration * 100) / kIterations));
+ DISPLAYUPDATE(lastUpdateTime,
+ 2, "\r%u%% ",
+ (unsigned)((iteration * 100) / kIterations));
++iteration;
}
COVER_best_wait(&best);
FASTCOVER_ctx_destroy(&ctx);
}
- LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "\r%79s\r", "");
/* Fill the output buffer and parameters with output of the best parameters */
{
const size_t dictSize = best.dictSize;