/*
Buffered version of Zstd compression library
- Copyright (C) 2015, Yann Collet.
+ Copyright (C) 2015-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - zstd source repository : https://github.com/Cyan4973/zstd
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+ - zstd homepage : http://www.zstd.net/
*/
#ifndef ZSTD_BUFFERED_H
#define ZSTD_BUFFERED_H
/* The objects defined into this file should be considered experimental.
- * They are not labelled stable, as their prototype may change in the future.
+ * They are not considered stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
*/
#endif
/* *************************************
-* Includes
+* Dependencies
***************************************/
#include <stddef.h> /* size_t */
ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
-/** ************************************************
+/*-*************************************************
* Streaming compression
*
* A ZBUFF_CCtx object is required to track streaming operation.
ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
+ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
+ void* dst, size_t* dstCapacityPtr,
+ const void* src, size_t* srcSizePtr);
-/** ************************************************
+/*-***************************************************************************
* Streaming decompression
*
-* A ZBUFF_DCtx object is required to track streaming operation.
+* A ZBUFF_DCtx object is required to track streaming operations.
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
* Use ZBUFF_decompressInit() to start a new decompression operation,
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
* or 0 when a frame is completely decoded
* or an error code, which can be tested using ZBUFF_isError().
*
-* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize
-* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
-* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
-* **************************************************/
+* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() / ZBUFF_recommendedDOutSize()
+* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
+* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
+* *******************************************************************************/
/* *************************************
ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
-/** The below functions provide recommended buffer sizes for Compression or Decompression operations.
+/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, and tend to offer better latency */
ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);
/*
zstd - buffered version of compression library
experimental complementary API, for static linking only
- Copyright (C) 2015, Yann Collet.
+ Copyright (C) 2015-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - zstd source repository : https://github.com/Cyan4973/zstd
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+ - zstd homepage : http://www.zstd.net
*/
#ifndef ZSTD_BUFFERED_STATIC_H
#define ZSTD_BUFFERED_STATIC_H
}
-static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
- const char* dstFileName, const char* srcFileName)
+static FILE* FIO_openSrcFile(const char* srcFileName)
{
+ FILE* f;
+
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n");
- *fileInPtr = stdin;
+ f = stdin;
SET_BINARY_MODE(stdin);
} else {
- *fileInPtr = fopen(srcFileName, "rb");
+ f = fopen(srcFileName, "rb");
}
- if ( *fileInPtr==0 ) {
- DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName);
- return 1;
- }
+ if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: No such file\n", srcFileName);
+
+ return f;
+}
+
+
+static FILE* FIO_openDstFile(const char* dstFileName)
+{
+ FILE* f;
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n");
- *fileOutPtr = stdout;
+ f = stdout;
SET_BINARY_MODE(stdout);
} else {
if (!g_overwrite) { /* Check if destination file already exists */
- *fileOutPtr = fopen( dstFileName, "rb" );
- if (*fileOutPtr != 0) { /* dest file exists, prompt for overwrite authorization */
- fclose(*fileOutPtr);
- DISPLAY("Warning : %s already exists \n", dstFileName);
- if ((g_displayLevel <= 1) || (*fileInPtr == stdin)) {
+ f = fopen( dstFileName, "rb" );
+ if (f != 0) { /* dest file exists, prompt for overwrite authorization */
+ fclose(f);
+ if (g_displayLevel <= 1) {
/* No interaction possible */
- DISPLAY("Operation aborted : %s already exists \n", dstFileName);
- return 1;
+ DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName);
+ return 0;
}
- DISPLAY("Overwrite ? (y/N) : ");
+ DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
{
int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
- DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
- return 1;
+ DISPLAY(" not overwritten \n");
+ return 0;
}
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
} } }
- *fileOutPtr = fopen( dstFileName, "wb" );
+ f = fopen( dstFileName, "wb" );
}
+ return f;
+}
- if (*fileOutPtr==0) EXM_THROW(13, "Pb opening %s", dstFileName);
+
+static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
+ const char* dstFileName, const char* srcFileName)
+{
+ *fileInPtr = FIO_openSrcFile(srcFileName);
+ if ( *fileInPtr==0 ) {
+ DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName);
+ return 1;
+ }
+
+ *fileOutPtr = FIO_openDstFile(dstFileName);
+ if (*fileOutPtr==0) return 1;
return 0;
}
+
/*!FIO_loadFile
* creates a buffer, pointed by *bufferPtr,
* loads "filename" content into it
void* dictBuffer;
size_t dictBufferSize;
ZBUFF_CCtx* ctx;
+ FILE* dstFile;
} cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName)
}
-/*!
- * FIO_compressFilename_extRess() :
- * @result : 0 : compression completed correctly,
- * 1 : missing or pb opening srcFileName
+/*! FIO_compressFilename_internal() :
+ * same as FIO_compressFilename_extRess(), with ress.desFile already opened
+ * @return : 0 : compression completed correctly,
+ * 1 : missing or pb opening srcFileName
*/
-static int FIO_compressFilename_extRess(cRess_t ress,
- const char* dstFileName, const char* srcFileName,
- int cLevel)
+static int FIO_compressFilename_internal(cRess_t ress,
+ const char* dstFileName, const char* srcFileName,
+ int cLevel)
{
FILE* srcFile;
- FILE* dstFile;
+ FILE* dstFile = ress.dstFile;
U64 filesize = 0;
U64 compressedfilesize = 0;
size_t dictSize = ress.dictBufferSize;
size_t sizeCheck, errorCode;
/* File check */
- if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
+ srcFile = FIO_openSrcFile(srcFileName);
+ if (!srcFile) return 1; /* srcFile could not be opened */
/* init */
filesize = FIO_getFileSize(srcFileName) + dictSize;
filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
- {
- /* Compress (buffered streaming ensures appropriate formatting) */
+ { /* Compress using buffered streaming */
size_t usedInSize = inSize;
size_t cSize = ress.dstBufferSize;
size_t result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
compressedfilesize += cSize;
}
-
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
}
/* clean */
fclose(srcFile);
- if (fclose(dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
return 0;
}
+/*! FIO_compressFilename_extRess() :
+ * @return : 0 : compression completed correctly,
+ * 1 : missing or pb opening srcFileName
+ */
+static int FIO_compressFilename_extRess(cRess_t ress,
+ const char* dstFileName, const char* srcFileName,
+ int cLevel)
+{
+ int result;
+
+ ress.dstFile = FIO_openDstFile(dstFileName);
+ if (ress.dstFile==0) return 1;
+
+ result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
+
+ if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
+ return result;
+}
+
+
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel)
{
int missed_files = 0;
char* dstFileName = (char*)malloc(FNSPACE);
size_t dfnSize = FNSPACE;
- const size_t suffixSize = strlen(suffix);
+ const size_t suffixSize = suffix ? strlen(suffix) : 0;
cRess_t ress;
/* init */
ress = FIO_createCResources(dictFileName);
/* loop on each file */
- for (u=0; u<nbFiles; u++) {
- size_t ifnSize = strlen(inFileNamesTable[u]);
- if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
- strcpy(dstFileName, inFileNamesTable[u]);
- strcat(dstFileName, suffix);
-
- missed_files += FIO_compressFilename_extRess(ress, dstFileName, inFileNamesTable[u], compressionLevel);
+ if (suffix) {
+ for (u=0; u<nbFiles; u++) {
+ size_t ifnSize = strlen(inFileNamesTable[u]);
+ if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
+ strcpy(dstFileName, inFileNamesTable[u]);
+ strcat(dstFileName, suffix);
+ missed_files += FIO_compressFilename_extRess(ress, dstFileName,
+ inFileNamesTable[u], compressionLevel);
+ }
+ } else {
+ ress.dstFile = stdout;
+ for (u=0; u<nbFiles; u++)
+ missed_files += FIO_compressFilename_internal(ress, stdoutmark,
+ inFileNamesTable[u], compressionLevel);
+ if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
}
/* Close & Free */
* Multiple File functions
***************************************/
/** FIO_compressMultipleFilenames() :
+ if `suffix == NULL`, output is stdout.
@return : nb of missing files */
int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
const char* suffix,
const char* dictFileName, int compressionLevel);
/** FIO_decompressMultipleFilenames() :
- @result : nb of missing or skipped files */
+ @return : nb of missing or skipped files */
int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
const char* suffix,
const char* dictFileName);
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
-printf "\n**** frame concatenation **** "
+
+echo "\n**** frame concatenation **** "
echo "hello " > hello.tmp
echo "world!" > world.tmp
echo frame concatenation test completed
-echo "**** flush write error test **** "
+
+echo "\n**** flush write error test **** "
echo "echo foo | $ZSTD > /dev/full"
echo foo | $ZSTD > /dev/full && die "write error not detected!"
echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
-echo "*** dictionary tests *** "
+echo "\n**** dictionary tests **** "
./datagen > tmpDict
./datagen -g1M | md5sum > tmp1
-./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dv | md5sum > tmp2
+./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
diff -q tmp1 tmp2
-echo "*** multiple files tests *** "
+echo "\n**** multiple files tests **** "
./datagen -s1 > tmp1 2> /dev/null
./datagen -s2 -g100K > tmp2 2> /dev/null
./datagen -s3 -g1M > tmp3 2> /dev/null
$ZSTD -f tmp*
+echo "compress tmp* : "
ls -ls tmp*
rm tmp1 tmp2 tmp3
+echo "decompress tmp* : "
$ZSTD -df *.zst
ls -ls tmp*
+echo "compress tmp* into stdout : "
+$ZSTD -c tmp1 tmp2 tmp3 > tmpall
+ls -ls tmp*
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
rm tmp*
-echo "**** zstd round-trip tests **** "
+echo "\n**** zstd round-trip tests **** "
roundTripTest
roundTripTest '' 6
if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) return badusage(programName);
- /* No warning message in pipe mode (stdin + stdout) or multiple mode */
- if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
- if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1;
-
- /* user-selected output filename only possible with a single file */
- if ((outFileName) && (filenameIdx>1)) {
+ /* user-selected output filename, only possible with a single file */
+ if (outFileName && strcmp(outFileName,stdoutmark) && (filenameIdx>1)) {
DISPLAY("Too many files (%u) on the command line. \n", filenameIdx);
return filenameIdx;
}
+ /* No warning message in pipe mode (stdin + stdout) or multiple mode */
+ if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
+ if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1;
+
/* IO Stream/File */
FIO_setNotificationLevel(displayLevel);
if (decode) {
- if (outFileName)
+ if (filenameIdx==1)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
else
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName);
} else { /* compression */
- if (outFileName)
+ if (filenameIdx==1)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
else
- operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName, cLevel);
+ operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, forceStdout ? NULL : ZSTD_EXTENSION, dictFileName, cLevel);
}
_end:
projects / thirdparty / zstd.git / commitdiff
