support large skippable frames

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9f5ff403378700a5d1a2d2a118cfc054f6a146a5..f832e081a0df1b475d5d2bc68ea598896c89f71c 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -122,13 +122,12 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx)   /* hidden interface *
 }
 
 
-#define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); }
-
 /** ZSTD_checkParams() :
     ensure param values remain within authorized range.
     @return : 0, or an error code if one value is beyond authorized range */
 size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
 {
+#   define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); }
     CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
     CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
     CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);

diff --git a/programs/fileio.c b/programs/fileio.c
index b7b201e02929e4a04de1380c6640736e6ac6838c..1023009e53caa3c054ec89ed7d757903d0285634 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -20,7 +20,7 @@
  ***************************************/
 #ifndef ZSTD_LEGACY_SUPPORT
 /* LEGACY_SUPPORT :
- *  decompressor can decode older formats (starting from Zstd 0.1+) */
+ *  decompressor can decode older formats (starting from zstd 0.1+) */
 #  define ZSTD_LEGACY_SUPPORT 1
 #endif
 
@@ -613,22 +613,22 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
     while (1) {
         ZSTD_inBuffer  inBuff = { ress.srcBuffer, readSize, 0 };
         ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
-        size_t const toRead = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
-        if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead));
+        size_t const readSizeHint = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
+        if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
 
         /* Write block */
         storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips);
         frameSize += outBuff.pos;
         DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
 
-        if (toRead == 0) break;   /* end of frame */
+        if (readSizeHint == 0) break;   /* end of frame */
         if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
 
         /* Fill input buffer */
-        if (toRead > ress.srcBufferSize) EXM_THROW(38, "too large block");
-        readSize = fread(ress.srcBuffer, 1, toRead, finput);
-        if (readSize == 0) EXM_THROW(39, "Read error : premature end");
-    }
+        {   size_t const toRead = MIN(readSizeHint, ress.srcBufferSize);  /* support large skippable frames */
+            readSize = fread(ress.srcBuffer, 1, toRead, finput);
+            if (readSize < toRead) EXM_THROW(39, "Read error : premature end");
+    }   }
 
     FIO_fwriteSparseEnd(foutput, storedSkips);
 
@@ -686,7 +686,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
             if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
             break;   /* no more input */
         }
-        readSomething = 1;
+        readSomething = 1;   /* there is at least >= 4 bytes in srcFile */
         if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
         {   U32 const magic = MEM_readLE32(ress.srcBuffer);
             if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) & (magic != ZSTD_MAGICNUMBER)