Addressing comments, removing cyclic dependency with header file, updating tests

diff --git a/programs/fileio.c b/programs/fileio.c
index 7cbf0280e567f35ed4f233cb65958c6f2b9d4f6c..e59fb80f61b57fae2ac00ddf70d0ae5a4397d2f2 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -319,6 +319,8 @@ struct FIO_prefs_s {
     /* Computation resources preferences */
     unsigned memLimit;
     int nbWorkers;
+
+    int excludeCompressedFiles;
 };
 
 
@@ -359,6 +361,7 @@ FIO_prefs_t* FIO_createPreferences(void)
     ret->srcSizeHint = 0;
     ret->testMode = 0;
     ret->literalCompressionMode = ZSTD_lcm_auto;
+    ret->excludeCompressedFiles = 0;
     return ret;
 }
 
@@ -402,6 +405,8 @@ void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) {
     prefs->nbWorkers = nbWorkers;
 }
 
+void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; }
+
 void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
     if (blockSize && prefs->nbWorkers==0)
         DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
@@ -1425,6 +1430,18 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
     return result;
 }
 
+static const char *compressedFileExtensions[] = {
+    ZSTD_EXTENSION,
+    TZSTD_EXTENSION,
+    GZ_EXTENSION,
+    TGZ_EXTENSION,
+    LZMA_EXTENSION,
+    XZ_EXTENSION,
+    TXZ_EXTENSION,
+    LZ4_EXTENSION,
+    TLZ4_EXTENSION,
+    NULL
+};
 
 /*! FIO_compressFilename_srcFile() :
  *  @return : 0 : compression completed correctly,
@@ -1451,19 +1468,18 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
         return 1;
     }
 
-    ress.srcFile = FIO_openSrcFile(srcFileName);
-    if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
-
     /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
-    * YES => ZSTD will not compress the file.
+    * YES => ZSTD will skip compression of the file and will return 0.
     * NO => ZSTD will resume with compress operation.
     */
-    if (g_excludeCompressedFiles && UTIL_isCompressedFile(srcFileName)) {  /* precompressed file (--exclude-compressed). DO NOT COMPRESS */
+    if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) {
         DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
-        fclose(ress.srcFile);
-        ress.srcFile = NULL;
         return 0;
     }
+
+    ress.srcFile = FIO_openSrcFile(srcFileName);
+    if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
+
     result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);

diff --git a/programs/fileio.h b/programs/fileio.h
index af2c5d9d1f5445c8ccdfd252e4b94dd2cf9ad303..a7da089f67da795c43fa2b4f03d67a31e117b279 100644 (file)
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -93,6 +93,7 @@ void FIO_setLiteralCompressionMode(
 
 void FIO_setNoProgress(unsigned noProgress);
 void FIO_setNotificationLevel(int level);
+void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles);
 
 /*-*************************************
 *  Single File functions

diff --git a/programs/util.c b/programs/util.c
index f820795545cd598eb22c92ec24f64874bbb4f20b..7212f7b811e412bd0107d4389d8c09a4acfca238 100644 (file)
--- a/programs/util.c
+++ b/programs/util.c
@@ -331,24 +331,18 @@ YES => Skip the file (return 0)
 NO => return 1
 */
 
-int UTIL_isCompressedFile(const char *inputName)
+int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
 {
-    return compareExtensions(inputName,g_compressedFileExtensions);
-}
-
-int compareExtensions(const char* infilename, const char* extensionList[])
-{
-  int i=0;
-   while(*extensionList != NULL)
+   while(*extensionList!=NULL)
    {
-     const char* ext = strstr(infilename,extensionList[i]);
+     const char* ext = strstr(inputName,*extensionList);
      if(ext)
         return 1;
       ++extensionList;
-      i++;
    }
    return 0;
 }
+
 /*
  * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
  *                       and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).

diff --git a/programs/util.h b/programs/util.h
index deb70786aafac58ee268871e13cc911c7c091792..deaea032352215ebeefdadc15fd70ad325ea2f46 100644 (file)
--- a/programs/util.h
+++ b/programs/util.h
@@ -39,7 +39,6 @@ extern "C" {
 #endif
 #include <time.h>         /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
 #include "mem.h"          /* U32, U64 */
-#include "fileio.h"
 
 /*-************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
@@ -127,19 +126,6 @@ extern int g_utilDisplayLevel;
     typedef struct stat stat_t;
 #endif
 
-int g_excludeCompressedFiles;
-static const char *g_compressedFileExtensions[] = {
-    ZSTD_EXTENSION,
-    TZSTD_EXTENSION,
-    GZ_EXTENSION,
-    TGZ_EXTENSION,
-    LZMA_EXTENSION,
-    XZ_EXTENSION,
-    TXZ_EXTENSION,
-    LZ4_EXTENSION,
-    TLZ4_EXTENSION,
-    NULL
-};
 
 int UTIL_fileExist(const char* filename);
 int UTIL_isRegularFile(const char* infilename);
@@ -148,9 +134,7 @@ U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 int UTIL_compareStr(const void *p1, const void *p2);
-int UTIL_isCompressedFile(const char* infilename);
-int compareExtensions(const char* infilename, const char *extensionList[]);
-
+int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
 U32 UTIL_isFIFO(const char* infilename);
 U32 UTIL_isLink(const char* infilename);
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index a704a1abd44f54f61848a51b47ef8e8cb90092be..5463c019f816ff54f55d9aef836348455eb9e7df 100644 (file)
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -709,7 +709,7 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
                     if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
                     if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
-                    if (!strcmp(argument, "--exclude-compressed")) { g_excludeCompressedFiles = 1; continue; }
+                    if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
                     /* long commands with arguments */
 #ifndef ZSTD_NODICT
                     if (longCommandWArg(&argument, "--train-cover")) {

diff --git a/tests/playTests.sh b/tests/playTests.sh
index ca286071df997ccb11459e9364ee1df3b0ffcb73..cb703227b24e524f6172fba45bc5938186841086 100755 (executable)
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -224,6 +224,8 @@ sleep 5
 ./datagen $size > precompressedFilterTestDir/input.7
 ./datagen $size > precompressedFilterTestDir/input.8
 $ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+test ! -f input.5.zst.zst
+test ! -f input.6.zst.zst
 file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
 file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
 if [[ $file2timestamp -ge $file1timestamp ]]; then
@@ -232,6 +234,7 @@ else
   println "Test is not successful"
 fi
 println "Test completed"
+sleep 5
 
 println "test : file removal"
 $ZSTD -f --rm tmp