# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
+#ifdef _MSC_VER
+#define snprintf sprintf_s
+#endif
+
/* *************************************
* Constants
/* *************************************
-* Macros
+* console display
***************************************/
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
+
+
+/* *************************************
+* Exceptions
+***************************************/
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, "\n"); \
+ exit(error); \
+}
/* *************************************
return nSpan;
}
+static U64 BMK_getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+#endif
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+ return (U64)statbuf.st_size;
+}
+
/* ********************************************************
* Bench functions
**********************************************************/
typedef struct
{
- char* srcPtr;
+ const char* srcPtr;
size_t srcSize;
char* cPtr;
size_t cRoom;
#define MIN(a,b) ((a)<(b) ? (a) : (b))
-static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
+static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
+ const char* displayName, int cLevel,
+ const char** fileNames, U32 nbFiles)
{
const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
- const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
- blockParam_t* const blockTable = (blockParam_t*) malloc(nbBlocks * sizeof(blockParam_t));
- const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
+ const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
+ blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
+ const size_t maxCompressedSize = (size_t)maxNbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
const compressor_t compressor = ZSTD_compress;
- U64 crcOrig;
+ U64 crcOrig = XXH64(srcBuffer, srcSize, 0);
+ U32 nbBlocks = 0;
/* init */
- if (strlen(fileName)>16)
- fileName += strlen(fileName)-16;
+ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
/* Memory allocation & restrictions */
if (!compressedBuffer || !resultBuffer || !blockTable)
- {
- DISPLAY("\nError: not enough memory!\n");
- free(compressedBuffer);
- free(resultBuffer);
- free(blockTable);
- return 12;
- }
-
- /* Calculating input Checksum */
- crcOrig = XXH64(srcBuffer, srcSize, 0);
+ EXM_THROW(31, "not enough memory");
/* Init blockTable data */
{
- U32 i;
- size_t remaining = srcSize;
- char* srcPtr = (char*)srcBuffer;
+ U32 fileNb;
+ const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)resultBuffer;
- for (i=0; i<nbBlocks; i++)
+ for (fileNb=0; fileNb<nbFiles; fileNb++)
{
- size_t thisBlockSize = MIN(remaining, blockSize);
- blockTable[i].srcPtr = srcPtr;
- blockTable[i].cPtr = cPtr;
- blockTable[i].resPtr = resPtr;
- blockTable[i].srcSize = thisBlockSize;
- blockTable[i].cRoom = ZSTD_compressBound(thisBlockSize);
- srcPtr += thisBlockSize;
- cPtr += blockTable[i].cRoom;
- resPtr += thisBlockSize;
- remaining -= thisBlockSize;
+ U64 fileSize = (nbFiles>=2) ? BMK_getFileSize(fileNames[fileNb]) : srcSize;
+ size_t remaining = (size_t)fileSize;
+ U32 nbBlocksforThisFile = (U32)((fileSize + (blockSize-1)) / blockSize);
+ U32 blockEnd = nbBlocks + nbBlocksforThisFile;
+ for ( ; nbBlocks<blockEnd; nbBlocks++)
+ {
+ size_t thisBlockSize = MIN(remaining, blockSize);
+ blockTable[nbBlocks].srcPtr = srcPtr;
+ blockTable[nbBlocks].cPtr = cPtr;
+ blockTable[nbBlocks].resPtr = resPtr;
+ blockTable[nbBlocks].srcSize = thisBlockSize;
+ blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize);
+ srcPtr += thisBlockSize;
+ cPtr += blockTable[nbBlocks].cRoom;
+ resPtr += thisBlockSize;
+ remaining -= thisBlockSize;
+ }
}
}
U32 blockNb;
/* Compression */
- DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, fileName, (U32)srcSize);
+ DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize);
memset(compressedBuffer, 0xE5, maxCompressedSize);
nbLoops = 0;
cSize += blockTable[blockNb].cSize;
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
ratio = (double)srcSize / (double)cSize;
- DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
+ DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
#if 1
/* Decompression */
milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
- DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
+ DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
/* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0);
{
unsigned u;
unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
- DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
+ DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++)
{
- if (((BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u])
+ if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u])
{
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
}
if (crcOrig == crcCheck)
- DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
+ DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
}
- /* End cleaning */
+ /* clean up */
free(compressedBuffer);
free(resultBuffer);
return 0;
}
-static U64 BMK_GetFileSize(const char* infilename)
-{
- int r;
-#if defined(_MSC_VER)
- struct _stat64 statbuf;
- r = _stat64(infilename, &statbuf);
-#else
- struct stat statbuf;
- r = stat(infilename, &statbuf);
-#endif
- if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
- return (U64)statbuf.st_size;
-}
-
static size_t BMK_findMaxMem(U64 requiredMem)
{
size_t step = 64 MB;
return (size_t)(requiredMem - step);
}
-static int BMK_benchOneFile(const char* inFileName, int cLevel)
+static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
+ const char* displayName, int cLevel,
+ const char** fileNamesTable, unsigned nbFiles)
{
- FILE* inFile;
- U64 inFileSize;
- size_t benchedSize, readSize;
- void* srcBuffer;
- int result=0;
-
- /* Check file existence */
- inFile = fopen(inFileName, "rb");
- if (inFile == NULL)
+ if (cLevel < 0)
{
- DISPLAY("Pb opening %s\n", inFileName);
- return 11;
+ int l;
+ for (l=1; l <= -cLevel; l++)
+ BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileNamesTable, nbFiles);
+ return;
}
+ BMK_benchMem(srcBuffer, benchedSize, displayName, cLevel, fileNamesTable, nbFiles);
+}
- /* Memory allocation & restrictions */
- inFileSize = BMK_GetFileSize(inFileName);
- benchedSize = BMK_findMaxMem(inFileSize * 3) / 3;
- if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
- if (benchedSize < inFileSize)
- DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize >> 20));
- srcBuffer = malloc(benchedSize);
- if (!srcBuffer)
- {
- DISPLAY("\nError: not enough memory!\n");
- fclose(inFile);
- return 12;
- }
+static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
+{
+ U64 total = 0;
+ unsigned n;
+ for (n=0; n<nbFiles; n++)
+ total += BMK_getFileSize(fileNamesTable[n]);
+ return total;
+}
- /* Fill input buffer */
- DISPLAY("Loading %s... \r", inFileName);
- readSize = fread(srcBuffer, 1, benchedSize, inFile);
- fclose(inFile);
+static void BMK_loadFiles(void* buffer, size_t bufferSize, const char** fileNamesTable, unsigned nbFiles)
+{
+ BYTE* buff = (BYTE*)buffer;
+ size_t pos = 0;
+ unsigned n;
- if (readSize != benchedSize)
+ for (n=0; n<nbFiles; n++)
{
- DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
- free(srcBuffer);
- return 13;
+ size_t readSize;
+ U64 fileSize = BMK_getFileSize(fileNamesTable[n]);
+ FILE* f = fopen(fileNamesTable[n], "rb");
+ if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+ DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
+ if (fileSize > bufferSize-pos) fileSize = bufferSize-pos;
+ readSize = fread(buff+pos, 1, (size_t)fileSize, f);
+ if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+ pos += readSize;
+ fclose(f);
}
+}
+
+static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, int cLevel)
+{
+ void* srcBuffer;
+ size_t benchedSize;
+ U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles);
+ char mfName[20] = {0};
+ const char* displayName = NULL;
+
+ /* Memory allocation & restrictions */
+ benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
+ if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
+ if (benchedSize < totalSizeToLoad)
+ DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
+ srcBuffer = malloc(benchedSize);
+ if (!srcBuffer) EXM_THROW(12, "not enough memory");
+
+ /* Load input buffer */
+ BMK_loadFiles(srcBuffer, benchedSize, fileNamesTable, nbFiles);
/* Bench */
- if (cLevel<0)
- {
- int l;
- for (l=1; l <= -cLevel; l++)
- result = BMK_benchMem(srcBuffer, benchedSize, inFileName, l);
- }
- else
- result = BMK_benchMem(srcBuffer, benchedSize, inFileName, cLevel);
+ snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
+ if (nbFiles > 1) displayName = mfName;
+ else displayName = fileNamesTable[0];
+
+ BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, fileNamesTable, nbFiles);
/* clean up */
free(srcBuffer);
- DISPLAY("\n");
- return result;
}
-static int BMK_syntheticTest(int cLevel, double compressibility)
+static void BMK_syntheticTest(int cLevel, double compressibility)
{
+ char name[20] = {0};
size_t benchedSize = 10000000;
void* srcBuffer = malloc(benchedSize);
- int result=0;
- char name[20] = {0};
/* Memory allocation */
- if (!srcBuffer)
- {
- DISPLAY("\nError: not enough memory!\n");
- free(srcBuffer);
- return 12;
- }
+ if (!srcBuffer) EXM_THROW(21, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
/* Bench */
-#ifdef _MSC_VER
- sprintf_s(name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100));
-#else
- snprintf (name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100));
-#endif
- /* Bench */
- if (cLevel<0)
- {
- int l;
- for (l=1; l <= -cLevel; l++)
- result = BMK_benchMem(srcBuffer, benchedSize, name, l);
- }
- else
- result = BMK_benchMem(srcBuffer, benchedSize, name, cLevel);
+ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
+ BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, NULL, 1);
- /* End */
+ /* clean up */
free(srcBuffer);
- DISPLAY("\n");
- return result;
}
double compressibility = (double)g_compressibilityDefault / 100;
if (nbFiles == 0)
- {
BMK_syntheticTest(cLevel, compressibility);
- }
else
- {
- /* Loop for each file */
- unsigned fileIdx = 0;
- while (fileIdx<nbFiles)
- {
- BMK_benchOneFile(fileNamesTable[fileIdx], cLevel);
- fileIdx++;
- }
- }
+ BMK_benchFileTable(fileNamesTable, nbFiles, cLevel);
return 0;
}
projects / thirdparty / zstd.git / commitdiff
