Added --exclude-compressed flag feature that skips compression of precompressed files

diff --git a/programs/fileio.c b/programs/fileio.c
index 0365f144991fdfbbfa1955891c4ea9d20c3fb35a..5172aa54cf496678e670a23d6ebd4776f95c2145 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1453,7 +1453,12 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
 
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
-
+    if (g_excludeCompressedFiles && !UTIL_isPrecompressedFile(srcFileName)) {  /* precompressed file (--exclude-compressed). DO NOT COMPRESS */
+        DISPLAYLEVEL(4, "Precompressed file: %s \n", srcFileName);
+        fclose(ress.srcFile);
+        ress.srcFile = NULL;
+        return 0;
+    }
     result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);

diff --git a/programs/util.c b/programs/util.c
index 587058805d0bd36c7dae6f53184b0a2ffc0ba466..830f2039ef6fb31d7a28da43eac45a35e5f37548 100644 (file)
--- a/programs/util.c
+++ b/programs/util.c
@@ -326,6 +326,27 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
 
 #endif /* #ifdef _WIN32 */
 
+/* Check if the file is precompressed (.zst, .lz4, .gz, .xz).
+YES => Skip the file (return 0)
+NO => return 1
+*/
+int UTIL_isPrecompressedFile(const char *inputName)
+{
+    return compareExtensions(inputName,compressedFileExtensions);
+}
+
+int compareExtensions(const char* infilename, const char extensionList[4][10])
+{
+   int i=0;
+   //char* ext = strchr(infilename, '.');
+   for(i=0;i<4;i++)
+   {
+     char* ext = strstr(infilename,extensionList[i]);
+     if(ext)
+        return 0;
+   }
+   return 1;
+}
 /*
  * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
  *                       and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).

diff --git a/programs/util.h b/programs/util.h
index 71ba0d4fc3262c1705c3b2e06e93ccb7e6a4782b..0da6255c02a26084d4c74042b2f948f06962b8da 100644 (file)
--- a/programs/util.h
+++ b/programs/util.h
@@ -127,6 +127,8 @@ extern int g_utilDisplayLevel;
     typedef struct stat stat_t;
 #endif
 
+int g_excludeCompressedFiles;
+static const char compressedFileExtensions[4][10] = {".zst",".gz",".xz",".lz4"};
 
 int UTIL_fileExist(const char* filename);
 int UTIL_isRegularFile(const char* infilename);
@@ -135,6 +137,8 @@ U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 int UTIL_compareStr(const void *p1, const void *p2);
+int UTIL_isPrecompressedFile(const char* infilename);
+int compareExtensions(const char* infilename, const char extensionList[4][10]);
 
 U32 UTIL_isFIFO(const char* infilename);
 U32 UTIL_isLink(const char* infilename);

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 3d5c4280e64ba522924e4e20529e82cc03cfe73e..11116afc076cf02396d0223457c2c89ac6d6c363 100644 (file)
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -118,6 +118,7 @@ static int usage(const char* programName)
 #endif
     DISPLAY( " -D file: use `file` as Dictionary \n");
     DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
+    DISPLAY( "--exclude-compressed:  only compress files that are not previously compressed \n");
     DISPLAY( " -f     : overwrite output without prompting and (de)compress links \n");
     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
     DISPLAY( " -k     : preserve source file(s) (default) \n");
@@ -708,7 +709,7 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
                     if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
                     if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
-
+                    if (!strcmp(argument, "--exclude-compressed")) { g_excludeCompressedFiles = 1; continue; }
                     /* long commands with arguments */
 #ifndef ZSTD_NODICT
                     if (longCommandWArg(&argument, "--train-cover")) {

diff --git a/tests/playTests.sh b/tests/playTests.sh
index f68ee81a56808a4e7aafb95e252b6b4d01999817..0946c03269165357315ed7e779cfd7fdf5b7c880 100755 (executable)
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -215,6 +215,25 @@ $ZSTD tmp -c --compress-literals    -19      | $ZSTD -t
 $ZSTD -b --fast=1 -i0e1 tmp --compress-literals
 $ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
 
+println "test: --exclude-compressed flag"
+mkdir precompressedFilterTestDir
+./datagen $size > precompressedFilterTestDir/input.5
+./datagen $size > precompressedFilterTestDir/input.6
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+sleep 5
+./datagen $size > precompressedFilterTestDir/input.7
+./datagen $size > precompressedFilterTestDir/input.8
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
+file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
+if [[ $file2timestamp -ge $file1timestamp ]]; then
+  println "Test is successful. input.5.zst is not precompressed and therefore not compressed/modified again."
+else
+  println "Test is not successful"
+fi
+println "Test completed"
+sleep 5
+
 println "test : file removal"
 $ZSTD -f --rm tmp
 test ! -f tmp  # tmp should no longer be present