bench: added cli command `-S` to benchmark multiple files separately

Currently, all files are joined by default,
they are compressed separately but benchmarked together,
providing a single final result.

Benchmarking files separately make it possible to accurately measure difference for each file.
This is expected to be useful while tuning optimal parser.

diff --git a/programs/bench.c b/programs/bench.c
index c612739368f673c993787727bae08c6135567e14..e28bed7478d8763e3a1de48ef23e030772b3a79e 100644 (file)
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -135,6 +135,11 @@ void BMK_setRealTime(unsigned priority) {
     g_realTime = (priority>0);
 }
 
+static U32 g_separateFiles = 0;
+void BMK_setSeparateFiles(unsigned separate) {
+    g_separateFiles = (separate>0);
+}
+
 static U32 g_ldmFlag = 0;
 void BMK_setLdmFlag(unsigned ldmFlag) {
     g_ldmFlag = ldmFlag;
@@ -482,11 +487,11 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     return (size_t)(requiredMem);
 }
 
-static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
+static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
                             const char* displayName, int cLevel, int cLevelLast,
                             const size_t* fileSizes, unsigned nbFiles,
                             const void* dictBuffer, size_t dictBufferSize,
-                            ZSTD_compressionParameters *compressionParams)
+                            const ZSTD_compressionParameters* const compressionParams)
 {
     int l;
 
@@ -518,7 +523,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
     At most, fills `buffer` entirely */
 static void BMK_loadFiles(void* buffer, size_t bufferSize,
                           size_t* fileSizes,
-                          const char** fileNamesTable, unsigned nbFiles)
+                          const char* const * const fileNamesTable, unsigned nbFiles)
 {
     size_t pos = 0, totalSize = 0;
     unsigned n;
@@ -550,16 +555,17 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
     if (totalSize == 0) EXM_THROW(12, "no data to bench");
 }
 
-static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
-                               int cLevelLast, ZSTD_compressionParameters *compressionParams)
+static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
+                               const char* const dictFileName,
+                               int const cLevel, int const cLevelLast,
+                               const ZSTD_compressionParameters* const compressionParams)
 {
     void* srcBuffer;
     size_t benchedSize;
     void* dictBuffer = NULL;
     size_t dictBufferSize = 0;
-    size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+    size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
-    char mfName[20] = {0};
 
     if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
 
@@ -588,13 +594,26 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
     BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
 
     /* Bench */
-    snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
-    {   const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
-        BMK_benchCLevel(srcBuffer, benchedSize,
-                        displayName, cLevel, cLevelLast,
-                        fileSizes, nbFiles,
-                        dictBuffer, dictBufferSize, compressionParams);
-    }
+    if (g_separateFiles) {
+        const BYTE* srcPtr = (const BYTE*)srcBuffer;
+        U32 fileNb;
+        for (fileNb=0; fileNb<nbFiles; fileNb++) {
+            size_t const fileSize = fileSizes[fileNb];
+            BMK_benchCLevel(srcPtr, fileSize,
+                            fileNamesTable[fileNb], cLevel, cLevelLast,
+                            fileSizes+fileNb, 1,
+                            dictBuffer, dictBufferSize, compressionParams);
+            srcPtr += fileSize;
+        }
+    } else {
+        char mfName[20] = {0};
+        snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
+        {   const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
+            BMK_benchCLevel(srcBuffer, benchedSize,
+                            displayName, cLevel, cLevelLast,
+                            fileSizes, nbFiles,
+                            dictBuffer, dictBufferSize, compressionParams);
+    }   }
 
     /* clean up */
     free(srcBuffer);

diff --git a/programs/bench.h b/programs/bench.h
index f4d415488b0fc10d5ca4e16569b977b39d8f4081..597f2503671b26219c165b02f0311c49f2c79fd5 100644 (file)
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -25,6 +25,7 @@ void BMK_setBlockSize(size_t blockSize);
 void BMK_setNbThreads(unsigned nbThreads);
 void BMK_setRealTime(unsigned priority);
 void BMK_setNotificationLevel(unsigned level);
+void BMK_setSeparateFiles(unsigned separate);
 void BMK_setAdditionalParam(int additionalParam);
 void BMK_setDecodeOnlyMode(unsigned decodeFlag);
 void BMK_setLdmFlag(unsigned ldmFlag);

diff --git a/programs/util.h b/programs/util.h
index c5e4365d04d1520c7943a0d6ec24bf0bebb58c4a..e44d7459ddfdce3b6cf0dfc495ac704f63d97496 100644 (file)
--- a/programs/util.h
+++ b/programs/util.h
@@ -336,7 +336,7 @@ UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
 }
 
 
-UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
+UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
 {
     U64 total = 0;
     int error = 0;

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index d5cb6285c4c1baf970fb421b8a7170ae8ea7dc03..91af49e2484727bc1d42caeb4a0b22884c726590 100644 (file)
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -377,6 +377,7 @@ int main(int argCount, const char* argv[])
         lastCommand = 0,
         nbThreads = 1,
         setRealTimePrio = 0,
+        separateFiles = 0,
         ldmFlag = 0;
     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
     size_t blockSize = 0;
@@ -633,6 +634,12 @@ int main(int argCount, const char* argv[])
                         blockSize = readU32FromChar(&argument);
                         break;
 
+                        /* benchmark files separately (hidden option) */
+                    case 'S':
+                        argument++;
+                        separateFiles = 1;
+                        break;
+
 #endif   /* ZSTD_NOBENCH */
 
                         /* nb of threads (hidden option) */
@@ -751,6 +758,7 @@ int main(int argCount, const char* argv[])
     if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH
         BMK_setNotificationLevel(g_displayLevel);
+        BMK_setSeparateFiles(separateFiles);
         BMK_setBlockSize(blockSize);
         BMK_setNbThreads(nbThreads);
         BMK_setRealTime(setRealTimePrio);