#define SAMPLESIZE_MAX (128 KB)
#define RANDOM_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
#define RANDOM_MEMMULT 9
-static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
+static const size_t g_maxMemory = (sizeof(size_t) == 4) ?
+ (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
#define NOISELENGTH 32
/*-*************************************
* Commandline related functions
***************************************/
-static unsigned readU32FromChar(const char** stringPtr)
-{
+static unsigned readU32FromChar(const char** stringPtr){
const char errorMsg[] = "error: numeric value too large";
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
* @return 0 and doesn't modify *stringPtr otherwise.
*/
-static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
-{
+static unsigned longCommandWArg(const char** stringPtr, const char* longCommand){
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
if (result) *stringPtr += comSize;
* *bufferSizePtr is modified, it provides the amount data loaded within buffer.
* sampleSizes is filled with the size of each sample.
*/
-static unsigned loadFiles(void* buffer, size_t* bufferSizePtr,
- size_t* sampleSizes, unsigned sstSize,
- const char** fileNamesTable, unsigned nbFiles, size_t targetChunkSize,
- unsigned displayLevel)
-{
+static unsigned loadFiles(void* buffer, size_t* bufferSizePtr, size_t* sampleSizes,
+ unsigned sstSize, const char** fileNamesTable, unsigned nbFiles,
+ size_t targetChunkSize, unsigned displayLevel) {
char* const buff = (char*)buffer;
size_t pos = 0;
unsigned nbLoadedChunks = 0, fileIndex;
/*-********************************************************
* Dictionary training functions
**********************************************************/
-static size_t findMaxMem(unsigned long long requiredMem)
-{
+static size_t findMaxMem(unsigned long long requiredMem) {
size_t const step = 8 MB;
void* testmem = NULL;
}
static void saveDict(const char* dictFileName,
- const void* buff, size_t buffSize)
-{
+ const void* buff, size_t buffSize) {
FILE* const f = fopen(dictFileName, "wb");
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
* provides the amount of data to be loaded and the resulting nb of samples.
* This is useful primarily for allocation purpose => sample buffer, and sample sizes table.
*/
-static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, unsigned displayLevel)
-{
+static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles,
+ size_t chunkSize, unsigned displayLevel) {
fileStats fs;
unsigned n;
memset(&fs, 0, sizeof(fs));
return fs;
}
-int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, unsigned maxDictSize,
- ZDICT_random_params_t *params){
+int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info,
+ unsigned maxDictSize,
+ ZDICT_random_params_t *params) {
unsigned const displayLevel = params->zParams.notificationLevel;
void* const dictBuffer = malloc(maxDictSize);
return result;
}
-sampleInfo* getSampleInfo(const char** fileNamesTable,
- unsigned nbFiles, size_t chunkSize, unsigned maxDictSize, const unsigned displayLevel){
+sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
+ unsigned maxDictSize, const unsigned displayLevel) {
fileStats const fs = getFileStats(fileNamesTable, nbFiles, chunkSize, displayLevel);
size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t));
size_t const memMult = RANDOM_MEMMULT;
/* Load input buffer */
DISPLAYLEVEL(3, "Shuffling input files\n");
shuffle(fileNamesTable, nbFiles);
- nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
+ nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples,
+ fileNamesTable, nbFiles, chunkSize, displayLevel);
sampleInfo *info = (sampleInfo *)malloc(sizeof(sampleInfo));
unsigned fileNamesNb = filenameIdx;
int followLinks = 0;
const char** extendedFileList = NULL;
- extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks);
+ extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf,
+ &fileNamesNb, followLinks);
if (extendedFileList) {
unsigned u;
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
* Check the validity of the parameters.
* Returns non-zero if the parameters are valid and 0 otherwise.
*/
-static int RANDOM_checkParameters(ZDICT_random_params_t parameters, size_t maxDictSize) {
+static int RANDOM_checkParameters(ZDICT_random_params_t parameters,
+ size_t maxDictSize) {
/* k is a required parameter */
if (parameters.k == 0) {
return 0;
/*! ZDICT_trainFromBuffer_random():
* Train a dictionary from an array of samples using the RANDOM algorithm.
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
- * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ * supplied with an array of sizes `samplesSizes`, providing the size of each
+ * sample, in order.
* The resulting dictionary will be saved into `dictBuffer`.
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
* or an error code, which can be tested with ZDICT_isError().
DISPLAYLEVEL(2, "Building dictionary\n");
{
- const size_t tail = RANDOM_buildDictionary(totalSamplesSize, samples, dictBuffer, dictBufferCapacity, parameters);
+ const size_t tail = RANDOM_buildDictionary(totalSamplesSize, samples,
+ dictBuffer, dictBufferCapacity, parameters);
const size_t dictSize = ZDICT_finalizeDictionary(
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
samplesBuffer, samplesSizes, nbSamples, parameters.zParams);