zbuff supports byte-by-byte decompression scenarios

diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c
index faa813707014302163aac571e8e7eca54b8de835..858b442f72edebb1b368164740fc74f47b88868d 100644 (file)
--- a/lib/decompress/zbuff_decompress.c
+++ b/lib/decompress/zbuff_decompress.c
@@ -64,14 +64,14 @@
 *           just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
 * *******************************************************************************/
 
-typedef enum { ZBUFFds_init, ZBUFFds_readHeader,
+typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
                ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
 
 /* *** Resource management *** */
 struct ZBUFF_DCtx_s {
     ZSTD_DCtx* zd;
     ZSTD_frameParams fParams;
-    size_t blockSize;
+    ZBUFF_dStage stage;
     char*  inBuff;
     size_t inBuffSize;
     size_t inPos;
@@ -79,7 +79,9 @@ struct ZBUFF_DCtx_s {
     size_t outBuffSize;
     size_t outStart;
     size_t outEnd;
-    ZBUFF_dStage stage;
+    size_t blockSize;
+    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+    size_t lhSize;
 };   /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
 
 
@@ -108,8 +110,8 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
 
 size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
 {
-    zbd->stage = ZBUFFds_readHeader;
-    zbd->inPos = zbd->outStart = zbd->outEnd = 0;
+    zbd->stage = ZBUFFds_loadHeader;
+    zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
     return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
 }
 
@@ -139,15 +141,29 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
         case ZBUFFds_init :
             return ERROR(init_missing);
 
-        case ZBUFFds_readHeader :
-            /* read header from src */
-            {   size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr);
-                if (ZSTD_isError(headerSize)) return headerSize;
-                if (headerSize) {
-                    /* not enough input to decode header : needs headerSize > *srcSizePtr */
-                    *dstCapacityPtr = 0;
-                    *srcSizePtr = 0;
-                    return headerSize;
+        case ZBUFFds_loadHeader :
+            {   size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
+                if (hSize != 0) {
+                    size_t const toLoad = hSize - zbd->lhSize;   /* if hSize!=0, hSize > zbd->lhSize */
+                    if (ZSTD_isError(hSize)) return hSize;
+                    if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
+                        memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
+                        zbd->lhSize += iend-ip; ip = iend; notDone = 0;
+                        *dstCapacityPtr = 0;
+                        return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
+                    }
+                    memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
+                    break;
+            }   }
+
+            /* Consume header */
+            {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);  /* == ZSTD_frameHeaderSize_min */
+                size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
+                if (ZSTD_isError(h1Result)) return h1Result;
+                if (h1Size < zbd->lhSize) {   /* long header */
+                    size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
+                    size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
+                    if (ZSTD_isError(h2Result)) return h2Result;
             }   }
 
             /* Frame header instruct buffer sizes */
@@ -175,8 +191,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
                     notDone = 0;
                     break;
                 }
-                if ((size_t)(iend-ip) >= neededInSize) {
-                    /* directly decode from src */
+                if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
                     size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
                         zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
                         ip, neededInSize);
@@ -200,6 +215,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
                 ip += loadedSize;
                 zbd->inPos += loadedSize;
                 if (loadedSize < toLoad) { notDone = 0; break; }   /* not enough input, wait for more */
+
                 /* decode loaded input */
                 {   size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
                         zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 72e52a450b7feb3f61e6aa0cae8d1a2931d178a1..7263d29bc42b026272fbf7306eaea862b53e5597 100644 (file)
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -657,7 +657,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
 }
 
 
-FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
+//FORCE_INLINE
+size_t ZSTD_execSequence(BYTE* op,
                                 BYTE* const oend, seq_t sequence,
                                 const BYTE** litPtr, const BYTE* const litLimit_8,
                                 const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
@@ -964,31 +965,29 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
     return dctx->expected;
 }
 
-size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     /* Sanity check */
     if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
-    ZSTD_checkContinuity(dctx, dst);
+    if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
 
     /* Decompress : frame header; part 1 */
     switch (dctx->stage)
     {
     case ZSTDds_getFrameHeaderSize :
-        {
-            if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);   /* impossible */
-            dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
-            if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
-            memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
-            if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
-                dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
-                dctx->stage = ZSTDds_decodeFrameHeader;
-                return 0;
-            }
-            dctx->expected = 0;   /* not necessary to copy more */
+        if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);   /* impossible */
+        dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
+        if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
+        memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
+        if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
+            dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
+            dctx->stage = ZSTDds_decodeFrameHeader;
+            return 0;
         }
+        dctx->expected = 0;   /* not necessary to copy more */
+
     case ZSTDds_decodeFrameHeader:
-        {
-            size_t result;
+        {   size_t result;
             memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
             result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
             if (ZSTD_isError(result)) return result;
@@ -997,8 +996,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
             return 0;
         }
     case ZSTDds_decodeBlockHeader:
-        {
-            blockProperties_t bp;
+        {   blockProperties_t bp;
             size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
             if (ZSTD_isError(cBlockSize)) return cBlockSize;
             if (bp.blockType == bt_end) {
@@ -1012,16 +1010,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
             return 0;
         }
     case ZSTDds_decompressBlock:
-        {
-            /* Decompress : block content */
-            size_t rSize;
+        {   size_t rSize;
             switch(dctx->bType)
             {
             case bt_compressed:
-                rSize = ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize);
+                rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
                 break;
             case bt_raw :
-                rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
+                rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
                 break;
             case bt_rle :
                 return ERROR(GENERIC);   /* not yet handled */

diff --git a/programs/fileio.c b/programs/fileio.c
index 36b1ff297d1f472764dec777ea7d6aa3edbd28e6..dea0869029071cace895f2419ab1c75765ae32a2 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -533,9 +533,9 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
 
     ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
 
-    /* Complete Header loading */
-    {   size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded;   /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */
-        size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);   /* can be <= toLoad (null string) */
+    /* Header loading (optional, saves one loop) */
+    {   size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded;   /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */
+        size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
         readSize = alreadyLoaded + loadedSize;
     }
 

diff --git a/programs/zbufftest.c b/programs/zbufftest.c
index f44f5d5e5f9834df89fb8ea81d3d90839301d73b..f1fa842ca9befd7bba2f233e1d8e3385ee85843b 100644 (file)
--- a/programs/zbufftest.c
+++ b/programs/zbufftest.c
@@ -188,6 +188,33 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "OK \n");
     }
 
+    /* Byte-by-byte decompression test */
+    DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
+    {   size_t r = 1, pIn=0, pOut=0;
+        while (r) {
+            size_t inS = 1;
+            size_t outS = 1;
+            r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
+            pIn += inS;
+            pOut += outS;
+        }
+        readSize = pIn;
+        genSize = pOut;
+    }
+    if (genSize != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
+    if (readSize != cSize) goto _output_error;   /* should have read the entire frame */
+    DISPLAYLEVEL(4, "OK \n");
+
+    /* check regenerated data is byte exact */
+    {   size_t i;
+        DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+        for (i=0; i<CNBufferSize; i++) {
+            if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
+        }
+        DISPLAYLEVEL(4, "OK \n");
+    }
+
 _end:
     ZBUFF_freeCCtx(zc);
     ZBUFF_freeDCtx(zd);