From: Yann Collet Date: Mon, 3 Jul 2017 17:27:16 +0000 (-0700) Subject: fileio : decoding malformed lzma frame does no longer exit() X-Git-Tag: v1.3.0~1^2~3^2~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c34185df13ded25bef0fa72879d41ad6dbae849a;p=thirdparty%2Fzstd.git fileio : decoding malformed lzma frame does no longer exit() makes it possible to continue decoding file list --- diff --git a/programs/fileio.c b/programs/fileio.c index 779437175..780c29b2f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -107,6 +107,17 @@ static clock_t g_time = 0; /*-************************************* * Errors ***************************************/ +/*-************************************* +* Debug +***************************************/ +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + #ifndef ZSTD_DEBUG # define ZSTD_DEBUG 0 #endif @@ -1347,10 +1358,10 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ /** FIO_decompressFrame() : - @return : size of decoded frame, or an error code + * @return : size of decoded zstd frame, or an error code */ -#define FIO_ERROR_ZSTD_DECODING ((unsigned long long)(-2)) -unsigned long long FIO_decompressFrame(dRess_t* ress, +#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) +unsigned long long FIO_decompressZstdFrame(dRess_t* ress, FILE* finput, const char* srcFileName, U64 alreadyDecoded) @@ -1375,7 +1386,7 @@ unsigned long long FIO_decompressFrame(dRess_t* ress, if (ZSTD_isError(readSizeHint)) { DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", srcFileName, ZSTD_getErrorName(readSizeHint)); - return FIO_ERROR_ZSTD_DECODING; + return FIO_ERROR_FRAME_DECODING; } /* Write block */ @@ -1393,7 +1404,7 @@ unsigned long long FIO_decompressFrame(dRess_t* ress, if (inBuff.size != inBuff.pos) { DISPLAYLEVEL(1, "%s : Decoding error (37) : should consume entire input \n", srcFileName); - return FIO_ERROR_ZSTD_DECODING; + return FIO_ERROR_FRAME_DECODING; } /* Fill input buffer */ @@ -1404,7 +1415,7 @@ unsigned long long FIO_decompressFrame(dRess_t* ress, if (ress->srcBufferLoaded < toRead) { DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", srcFileName); - return FIO_ERROR_ZSTD_DECODING; + return FIO_ERROR_FRAME_DECODING; } } } FIO_fwriteSparseEnd(ress->dstFile, storedSkips); @@ -1444,7 +1455,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, strm.avail_in = (uInt)ress->srcBufferLoaded; } ret = inflate(&strm, flush); - if (ret == Z_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName); + if (ret == Z_BUF_ERROR) EXM_THROW(89, "zstd: %s: premature gz end", srcFileName); if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; @@ -1476,19 +1487,23 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, unsigned long long outFileSize = 0; lzma_stream strm = LZMA_STREAM_INIT; lzma_action action = LZMA_RUN; - lzma_ret ret; + lzma_ret initRet; + int decodingError = 0; strm.next_in = 0; strm.avail_in = 0; if (plain_lzma) { - ret = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ + initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ } else { - ret = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ + initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ } - if (ret != LZMA_OK) - EXM_THROW(71, "zstd: %s: lzma_alone_decoder/lzma_stream_decoder error %d", - srcFileName, ret); + if (initRet != LZMA_OK) { + DISPLAYLEVEL(1, "zstd: %s: %s error %d \n", + plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder", + srcFileName, initRet); + return FIO_ERROR_FRAME_DECODING; + } strm.next_out = (BYTE*)ress->dstBuffer; strm.avail_out = ress->dstBufferSize; @@ -1496,6 +1511,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, strm.avail_in = ress->srcBufferLoaded; for ( ; ; ) { + lzma_ret ret; if (strm.avail_in == 0) { ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; @@ -1504,16 +1520,21 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, } ret = lzma_code(&strm, action); - if (ret == LZMA_BUF_ERROR) - EXM_THROW(39, "zstd: %s: premature end", srcFileName); + if (ret == LZMA_BUF_ERROR) { + DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName); + decodingError = 1; break; + } if (ret != LZMA_OK && ret != LZMA_STREAM_END) { - DISPLAY("zstd: %s: lzma_code decoding error %d \n", srcFileName, ret); - return 0; + DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n", + srcFileName, ret); + decodingError = 1; break; } { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { - if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) - EXM_THROW(31, "Write error : cannot write to output file"); + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { + DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); + decodingError = 1; break; + } outFileSize += decompBytes; strm.next_out = (BYTE*)ress->dstBuffer; strm.avail_out = ress->dstBufferSize; @@ -1524,7 +1545,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in); ress->srcBufferLoaded = strm.avail_in; lzma_end(&strm); - return outFileSize; + return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } #endif @@ -1596,45 +1617,37 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, -/** FIO_decompressSrcFile() : - Decompression `srcFileName` into `ress.dstFile` - @return : 0 : OK - 1 : operation not started -*/ -static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName) +/** FIO_decompressFrames() : + * Find and decode frames inside srcFile + * srcFile presumed opened and valid + * @return : 0 : OK + * 1 : error + */ +static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, + const char* dstFileName, const char* srcFileName) { - FILE* srcFile; unsigned readSomething = 0; unsigned long long filesize = 0; - - if (UTIL_isDirectory(srcFileName)) { - DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); - return 1; - } - - srcFile = FIO_openSrcFile(srcFileName); - if (srcFile==NULL) return 1; + assert(srcFile != NULL); /* for each frame */ for ( ; ; ) { /* check magic number -> version */ size_t const toRead = 4; - const BYTE* buf = (const BYTE*)ress.srcBuffer; + const BYTE* const buf = (const BYTE*)ress.srcBuffer; if (ress.srcBufferLoaded < toRead) ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, (size_t)1, toRead - ress.srcBufferLoaded, srcFile); if (ress.srcBufferLoaded==0) { if (readSomething==0) { - DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); - fclose(srcFile); + DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName); return 1; - } /* srcFileName is empty */ + } /* else, srcFileName is just empty */ break; /* no more input */ } - readSomething = 1; /* there is at least >= 4 bytes in srcFile */ + readSomething = 1; /* there is at least 1 byte in srcFile */ if (ress.srcBufferLoaded < toRead) { - DISPLAY("zstd: %s: unknown header \n", srcFileName); - fclose(srcFile); + DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); return 1; } if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ @@ -1643,17 +1656,17 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch if (result == 0) return 1; filesize += result; #else - DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); + DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName); return 1; #endif } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ #ifdef ZSTD_LZMADECOMPRESS - unsigned long long const result = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); - if (result == 0) return 1; - filesize += result; + unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); + if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; + filesize += frameSize; #else - DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without ZSTD_LZMADECOMPRESS) -- ignored \n", srcFileName); + DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName); return 1; #endif } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { @@ -1662,41 +1675,59 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch if (result == 0) return 1; filesize += result; #else - DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without ZSTD_LZ4DECOMPRESS) -- ignored \n", srcFileName); + DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); return 1; #endif + } else if (ZSTD_isFrame(buf, toRead)) { + unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, srcFileName, filesize); + if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; + filesize += frameSize; + } else if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ + return FIO_passThrough(ress.dstFile, srcFile, + ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); } else { - if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { - if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ - unsigned const result = FIO_passThrough(ress.dstFile, srcFile, - ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); - if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ - return result; - } else { - DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); - fclose(srcFile); - return 1; - } } - { unsigned long long const frameSize = FIO_decompressFrame(&ress, srcFile, srcFileName, filesize); - if (frameSize == FIO_ERROR_ZSTD_DECODING) { - fclose(srcFile); - return 1; - } - filesize += frameSize; - } } - } /* for each frame */ + DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); + return 1; + } } /* for each frame */ /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); + return 0; +} + + +/** FIO_decompressSrcFile() : + Decompression `srcFileName` into `ress.dstFile` + @return : 0 : OK + 1 : operation not started +*/ +static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName) +{ + FILE* srcFile; + int result; + + if (UTIL_isDirectory(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); + return 1; + } + + srcFile = FIO_openSrcFile(srcFileName); + if (srcFile==NULL) return 1; + + result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName); + /* Close file */ - if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ + if (fclose(srcFile)) { + EXM_THROW(33, "zstd: %s: %s", srcFileName, strerror(errno)); /* error should never happen */ + return 1; + } if (g_removeSrcFile /* --rm */ && strcmp(srcFileName, stdinmark)) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); } - return 0; + return result; }