From: Yann Collet Date: Tue, 26 Nov 2019 19:20:26 +0000 (-0800) Subject: Merge branch 'dev' into ahmed_file X-Git-Tag: v1.4.5^2~141^2~9 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c71bd45a3b25f1165c90bf817d6147d5b886fe46;p=thirdparty%2Fzstd.git Merge branch 'dev' into ahmed_file --- c71bd45a3b25f1165c90bf817d6147d5b886fe46 diff --cc programs/util.c index 3ca404b34,cb177a3c7..bb511bd5d --- a/programs/util.c +++ b/programs/util.c @@@ -24,49 -24,20 +24,59 @@@ extern "C" #include /* needed for _mkdir in windows */ #endif +#if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ +# include /* opendir, readdir */ +# include /* strerror, memcpy */ +#endif /* #ifdef _WIN32 */ + + +/*-**************************************** +* Internal Macros +******************************************/ + +/* CONTROL is like an assert(), but is never disabled. + * Since it's always active, it can trigger side effects. + */ +#define CONTROL(c) { \ + if (!(c)) { \ + UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \ + __FILE__, __LINE__, #c); \ + exit(1); \ +} } + - +/* + * A modified version of realloc(). + * If UTIL_realloc() fails the original block is freed. + */ +UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size) +{ + void *newptr = realloc(ptr, size); + if (newptr) return newptr; + free(ptr); + return NULL; +} + + #if defined(_MSC_VER) + #define chmod _chmod + #endif + +/*-**************************************** +* Console log +******************************************/ +int g_utilDisplayLevel; + + - /*-**************************************** - * Public API - ******************************************/ + /*-************************************* + * Constants + ***************************************/ + #define LIST_SIZE_INCREASE (8*1024) ++#define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50 + + + /*-************************************* + * Functions + ***************************************/ int UTIL_fileExist(const char* filename) { diff --cc programs/util.h index 14e5b4948,e67faba5c..0065ec46f --- a/programs/util.h +++ b/programs/util.h @@@ -129,28 -123,35 +123,26 @@@ extern int g_utilDisplayLevel int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); - int UTIL_setFileStat(const char* filename, stat_t* statbuf); - U32 UTIL_isDirectory(const char* infilename); - int UTIL_getFileStat(const char* infilename, stat_t* statbuf); + int UTIL_isDirectory(const char* infilename); int UTIL_isSameFile(const char* file1, const char* file2); - int UTIL_compareStr(const void *p1, const void *p2); int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]); - const char* UTIL_getFileExtension(const char* infilename); + int UTIL_isLink(const char* infilename); + int UTIL_isFIFO(const char* infilename); - #ifndef _MSC_VER - U32 UTIL_isFIFO(const char* infilename); - #endif - U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) U64 UTIL_getFileSize(const char* infilename); - - U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles); + U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles); + int UTIL_getFileStat(const char* infilename, stat_t* statbuf); + int UTIL_setFileStat(const char* filename, stat_t* statbuf); + int UTIL_chmod(char const* filename, mode_t permissions); /*< like chmod, but avoid changing permission of /dev/null */ + int UTIL_compareStr(const void *p1, const void *p2); + const char* UTIL_getFileExtension(const char* infilename); -/* - * A modified version of realloc(). - * If UTIL_realloc() fails the original block is freed. -*/ -UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size) -{ - void* const newptr = realloc(ptr, size); - if (newptr) return newptr; - free(ptr); - return NULL; -} +/*-**************************************** + * Lists of Filenames + ******************************************/ -int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks); #ifdef _WIN32 # define UTIL_HAS_CREATEFILELIST #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ diff --cc programs/zstdcli.c index 57dbaa821,ad2bbbd88..cf2f3682d --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@@ -1002,40 -1003,28 +1002,38 @@@ int main(int argCount, const char* argv #ifdef UTIL_HAS_CREATEFILELIST g_utilDisplayLevel = g_displayLevel; if (!followLinks) { - unsigned u; - for (u=0, fileNamesNb=0; utableSize; + for (u=0, fileNamesNb=0; ufileNames[u]) - #ifndef _MSC_VER - && !UTIL_isFIFO(filenames->fileNames[u]) - #endif /* _MSC_VER */ ++ if ( UTIL_isLink(filenames->fileNames[u]) ++ && !UTIL_isFIFO(filenames->fileNames[u]) ) { - DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenames->fileNames[u]); - DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]); ++ DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring \n", filenames->fileNames[u]); } else { - filenameTable[fileNamesNb++] = filenameTable[u]; + filenames->fileNames[fileNamesNb++] = filenames->fileNames[u]; } } - if (fileNamesNb == 0 && filenameIdx > 0) + if (fileNamesNb == 0 && nbFilenames > 0) /* all names are eliminated */ CLEAN_RETURN(1); - filenameIdx = fileNamesNb; + filenames->tableSize = fileNamesNb; + } /* if (!followLinks) */ + + /* read names from a file */ + if (file_of_names->tableSize) { + size_t const nbFileLists = file_of_names->tableSize; + size_t flNb; + for (flNb=0; flNb < nbFileLists; flNb++) { + FileNamesTable* const fnt = UTIL_createFileNamesTable_fromFileName(file_of_names->fileNames[flNb]); + if (fnt==NULL) { + DISPLAYLEVEL(1, "zstd: error reading %s \n", file_of_names->fileNames[flNb]); + CLEAN_RETURN(1); + } + filenames = UTIL_mergeFileNamesTable(filenames, fnt); + } } + if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ - extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks); - if (extendedFileList) { - unsigned u; - for (u=0; u decompression only tests " - head -c 1048576 /dev/zero > tmp + println "\n===> decompression only tests " + # the following test verifies that the decoder is compatible with RLE as first block + # older versions of zstd cli are not able to decode such corner case. + # As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions. + dd bs=1048576 count=1 if=/dev/zero of=tmp $ZSTD -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" $DIFF -s tmp1 tmp rm tmp* -println "\m===> compress multiple files" + - println "test : compress multiple files" ++println "\n===> compress multiple files" println hello > tmp1 println world > tmp2 $ZSTD tmp1 tmp2 -o "$INTOVOID" -f @@@ -301,34 -306,23 +308,49 @@@ $ZSTD tmp1 tmp2 -o tmpexists && die "sh if [ "$?" -eq 139 ]; then die "should not have segfaulted" fi +println "\n===> multiple files and shell completion " +./datagen -s1 > tmp1 2> $INTOVOID +./datagen -s2 -g100K > tmp2 2> $INTOVOID +./datagen -s3 -g1M > tmp3 2> $INTOVOID +println "compress tmp* : " +$ZSTD -f tmp* +test -f tmp1.zst +test -f tmp2.zst +test -f tmp3.zst +rm tmp1 tmp2 tmp3 +println "decompress tmp* : " +$ZSTD -df ./*.zst +test -f tmp1 +test -f tmp2 +test -f tmp3 +println "compress tmp* into stdout > tmpall : " +$ZSTD -c tmp1 tmp2 tmp3 > tmpall +test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) +println "decompress tmpall* into stdout > tmpdec : " +cp tmpall tmpall2 +$ZSTD -dc tmpall* > tmpdec +test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) +println "compress multiple files including a missing one (notHere) : " +$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" rm tmp* + - println "test : compress multiple files into an output directory, --output-dir-flat" + if [ -n "$DEVNULLRIGHTS" ] + then + # these tests requires sudo rights, which is uncommon. + # they are only triggered if DEVNULLRIGHTS macro is defined. + println "\n===> checking /dev/null permissions are unaltered " + ./datagen > tmp + sudo $ZSTD tmp -o $INTOVOID # sudo rights could modify /dev/null permissions + sudo $ZSTD tmp -c > $INTOVOID + $ZSTD tmp -f -o tmp.zst + sudo $ZSTD -d tmp.zst -c > $INTOVOID + sudo $ZSTD -d tmp.zst -o $INTOVOID + ls -las $INTOVOID | grep "rw-rw-rw-" + fi + ++ + println "\n===> compress multiple files into an output directory, --output-dir-flat" println henlo > tmp1 mkdir tmpInputTestDir mkdir tmpInputTestDir/we