Added : Frame concatenation ability

diff --git a/lib/fse.c b/lib/fse.c
index 8b1223dad049a1332d1cddfd9902d30ab9a5453c..4fc692f24e1b5baf0bd2ea2767f884456b034f23 100644 (file)
--- a/lib/fse.c
+++ b/lib/fse.c
@@ -914,8 +914,8 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
                 else
                 {
                     bitCount -= (int)(8 * (iend - 4 - ip));
-                                       ip = iend - 4;
-                               }
+                    ip = iend - 4;
+                }
                 bitStream = FSE_readLE32(ip) >> (bitCount & 31);
             }
         }
@@ -967,20 +967,20 @@ void  FSE_freeCTable (FSE_CTable* ct)
 /* provides the minimum logSize to safely represent a distribution */
 static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
 {
-       U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
-       U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
-       U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
-       return minBits;
+    U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
+    U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
+    U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
+    return minBits;
 }
 
 unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
 {
-       U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
+    U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
     U32 tableLog = maxTableLog;
-       U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
+    U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
     if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
-       if (maxBitsSrc < tableLog) tableLog = maxBitsSrc;   /* Accuracy can be reduced */
-       if (minBits > tableLog) tableLog = minBits;   /* Need a minimum to safely represent all symbol values */
+    if (maxBitsSrc < tableLog) tableLog = maxBitsSrc;   /* Accuracy can be reduced */
+    if (minBits > tableLog) tableLog = minBits;   /* Need a minimum to safely represent all symbol values */
     if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
     if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
     return tableLog;
@@ -1569,8 +1569,8 @@ size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits)   /* only if nbBits >=
 
 unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
 {
-       if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should never happen */
-               return FSE_DStream_tooFar;
+    if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should never happen */
+        return FSE_DStream_tooFar;
 
     if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
     {
@@ -1834,7 +1834,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32
         if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC;   /* not implemented (not possible with current format) */
         if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall;   /* not enough space within dst buffer */
         op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
-               huffWeight[maxSymbolValue] = 0;   /* to be sure it doesn't cause issue in final combination */
+        huffWeight[maxSymbolValue] = 0;   /* to be sure it doesn't cause issue in final combination */
         for (n=0; n<maxSymbolValue; n+=2)
             op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
         return ((maxSymbolValue+1)/2) + 1;
@@ -1878,7 +1878,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
             U32 rankLast[HUF_MAX_TABLELOG];
             U32 currentNbBits = maxNbBits;
             int pos;
-                       memset(rankLast, 0xF0, sizeof(rankLast));
+            memset(rankLast, 0xF0, sizeof(rankLast));
             for (pos=n ; pos >= 0; pos--)
             {
                 if (huffNode[pos].nbBits >= currentNbBits) continue;
@@ -1917,20 +1917,20 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
                 }
             }
 
-                       while (totalCost < 0)   /* Sometimes, cost correction overshoot */
-                       {
-                               if (rankLast[1] == noOne)   /* special case, no weight 1, let's find it back at n */
-                               {
-                                       while (huffNode[n].nbBits == maxNbBits) n--;
-                                       huffNode[n+1].nbBits--;
-                                       rankLast[1] = n+1;
-                                       totalCost++;
-                                       continue;
-                               }
-                               huffNode[ rankLast[1] + 1 ].nbBits--;
-                               rankLast[1]++;
-                               totalCost ++;
-                       }
+            while (totalCost < 0)   /* Sometimes, cost correction overshoot */
+            {
+                if (rankLast[1] == noOne)   /* special case, no weight 1, let's find it back at n */
+                {
+                    while (huffNode[n].nbBits == maxNbBits) n--;
+                    huffNode[n+1].nbBits--;
+                    rankLast[1] = n+1;
+                    totalCost++;
+                    continue;
+                }
+                huffNode[ rankLast[1] + 1 ].nbBits--;
+                rankLast[1]++;
+                totalCost ++;
+            }
         }
     }
 
@@ -1981,7 +1981,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
     /* safety checks */
     if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
     if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC;
-       memset(huffNode0, 0, sizeof(huffNode0));
+    memset(huffNode0, 0, sizeof(huffNode0));
 
     // sort, decreasing order
     HUF_sort(huffNode, count, maxSymbolValue);
@@ -2066,7 +2066,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size
     FSE_CStream_t bitC;
 
     /* init */
-       if (dstSize < 8) return 0;
+    if (dstSize < 8) return 0;
     op += 6;   /* jump Table -- could be optimized by delta / deviation */
     errorCode = FSE_initCStream(&bitC, op, oend-op);
     if (FSE_isError(errorCode)) return 0;

diff --git a/lib/zstd.c b/lib/zstd.c
index 23fdff73bd82066a3bd2486bbc17f3635d29d279..a574efc9ecce64fbffa32920d2961cd0efae2d32 100644 (file)
--- a/lib/zstd.c
+++ b/lib/zstd.c
@@ -542,11 +542,11 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
     const size_t minGain = ZSTD_minGain(srcSize);
     BYTE* const ostart = (BYTE*)dst;
     size_t hsize;
-       static const size_t LHSIZE = 5;
+    static const size_t LHSIZE = 5;
 
-       if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;   /* not enough space for compression */
+    if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;   /* not enough space for compression */
 
-       hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
+    hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
     if (hsize<2) return hsize;   /* special cases */
     if (hsize >= srcSize - minGain) return 0;
 
@@ -619,8 +619,8 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
     }
 
     /* Sequences Header */
-       if ((oend-op) < 2+3+6)  /* nbSeq + dumpsLength + 3*rleCTable*/
-               return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+    if ((oend-op) < 2+3+6)  /* nbSeq + dumpsLength + 3*rleCTable*/
+        return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
     ZSTD_writeLE16(op, (U16)nbSeq); op+=2;
     seqHead = op;
 
@@ -640,7 +640,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
             op[2] = (BYTE)(dumpsLength);
             op += 3;
         }
-               if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+        if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
         memcpy(op, seqStorePtr->dumpsStart, dumpsLength);
         op += dumpsLength;
     }
@@ -661,11 +661,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
     }
     else
     {
-               size_t NCountSize;
+        size_t NCountSize;
         tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
         FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
-               NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
-               if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+        NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+        if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
         op += NCountSize;
         FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
         LLtype = bt_compressed;
@@ -696,11 +696,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
     }
     else
     {
-               size_t NCountSize;
+        size_t NCountSize;
         tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
         FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
-               NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
-               if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+        NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+        if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
         op += NCountSize;
         FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
         Offtype = bt_compressed;
@@ -722,11 +722,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
     }
     else
     {
-               size_t NCountSize;
+        size_t NCountSize;
         tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
         FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
-               NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
-               if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+        NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+        if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
         op += NCountSize;
         FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
         MLtype = bt_compressed;
@@ -1028,7 +1028,7 @@ size_t ZSTD_compressContinue(ZSTD_Cctx*  cctx, void* dst, size_t maxDstSize, con
         size_t blockSize = BLOCKSIZE;
         if (blockSize > srcSize) blockSize = srcSize;
 
-               if (maxDstSize < 2*ZSTD_blockHeaderSize+1)  /* one RLE block + endMark */
+        if (maxDstSize < 2*ZSTD_blockHeaderSize+1)  /* one RLE block + endMark */
             return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
 
         /* update hash table */
@@ -1250,8 +1250,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
     U32 LLlog, Offlog, MLlog;
     size_t dumpsLength;
 
-       /* check */
-       if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
+    /* check */
+    if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
 
     /* SeqHead */
     *nbSeq = ZSTD_readLE16(ip); ip+=2;
@@ -1273,8 +1273,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
     *dumpsPtr = ip;
     ip += dumpsLength;
 
-       /* check */
-       if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
+    /* check */
+    if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
 
     /* sequences */
     {
@@ -1295,7 +1295,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
             max = MaxLL;
             headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
             if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
-                       if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
+            if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
             ip += headerSize;
             FSE_buildDTable(DTableLL, norm, max, LLlog);
         }
@@ -1314,7 +1314,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
             max = MaxOff;
             headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
             if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
-                       if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
+            if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
             ip += headerSize;
             FSE_buildDTable(DTableOffb, norm, max, Offlog);
         }
@@ -1333,7 +1333,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
             max = MaxML;
             headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
             if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
-                       if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
+            if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
             ip += headerSize;
             FSE_buildDTable(DTableML, norm, max, MLlog);
         }
@@ -1417,9 +1417,9 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
 
 
 static size_t ZSTD_execSequence(BYTE* op,
-                                                               seq_t sequence,
-                                                               const BYTE** litPtr, const BYTE* const litLimit,
-                                                               BYTE* const base, BYTE* const oend)
+                                seq_t sequence,
+                                const BYTE** litPtr, const BYTE* const litLimit,
+                                BYTE* const base, BYTE* const oend)
 {
     static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};   /* added */
     static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* substracted */
@@ -1430,7 +1430,7 @@ static size_t ZSTD_execSequence(BYTE* op,
 
     /* check */
     if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;   /* overwrite beyond dst buffer */
-       if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
+    if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
     if (sequence.matchLength > (size_t)(*litPtr-op))  return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;    /* overwrite literal segment */
 
     /* copy Literals */
@@ -1442,18 +1442,18 @@ static size_t ZSTD_execSequence(BYTE* op,
     *litPtr = litEnd;   /* update for next sequence */
 
     /* check : last match must be at a minimum distance of 8 from end of dest buffer */
-       if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+    if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
 
-       /* copy Match */
+    /* copy Match */
     {
         const U32 overlapRisk = (((size_t)(litEnd - endMatch)) < 12);
         const BYTE* match = op - sequence.offset;            /* possible underflow at op - offset ? */
         size_t qutt = 12;
         U64 saved[2];
 
-               /* check */
-               if (match < base) return (size_t)-ZSTD_ERROR_corruption;
-               if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
+        /* check */
+        if (match < base) return (size_t)-ZSTD_ERROR_corruption;
+        if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
 
         /* save beginning of literal sequence, in case of write overlap */
         if (overlapRisk)
@@ -1473,7 +1473,7 @@ static size_t ZSTD_execSequence(BYTE* op,
             ZSTD_copy4(op+4, match);
             match -= dec64;
         } else { ZSTD_copy8(op, match); }
-               op += 8; match += 8;
+        op += 8; match += 8;
 
         if (endMatch > oend-12)
         {
@@ -1498,11 +1498,11 @@ static size_t ZSTD_execSequence(BYTE* op,
 typedef struct ZSTD_Dctx_s
 {
     U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
-       U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
-       U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+    U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+    U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
     void* previousDstEnd;
     void* base;
-       size_t expected;
+    size_t expected;
     blockType_t bType;
     U32 phase;
 } dctx_t;
@@ -1514,7 +1514,7 @@ static size_t ZSTD_decompressSequences(
                          const void* seqStart, size_t seqSize,
                          const BYTE* litStart, size_t litSize)
 {
-       dctx_t* dctx = (dctx_t*)ctx;
+    dctx_t* dctx = (dctx_t*)ctx;
     const BYTE* ip = (const BYTE*)seqStart;
     const BYTE* const iend = ip + seqSize;
     BYTE* const ostart = (BYTE* const)dst;
@@ -1526,9 +1526,9 @@ static size_t ZSTD_decompressSequences(
     int nbSeq;
     const BYTE* dumps;
     U32* DTableLL = dctx->LLTable;
-       U32* DTableML = dctx->MLTable;
+    U32* DTableML = dctx->MLTable;
     U32* DTableOffb = dctx->OffTable;
-       BYTE* const base = (BYTE*) (dctx->base);
+    BYTE* const base = (BYTE*) (dctx->base);
 
     /* Build Decoding Tables */
     errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps,
@@ -1658,8 +1658,8 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
 
 size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
-       dctx_t ctx;
-       ctx.base = dst;
+    dctx_t ctx;
+    ctx.base = dst;
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
@@ -1668,14 +1668,20 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
 *  Streaming Decompression API
 *******************************/
 
+size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx)
+{
+    dctx->expected = ZSTD_frameHeaderSize;
+    dctx->phase = 0;
+    dctx->previousDstEnd = NULL;
+    dctx->base = NULL;
+    return 0;
+}
+
 ZSTD_Dctx* ZSTD_createDCtx(void)
 {
     ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
     if (dctx==NULL) return NULL;
-    dctx->expected = ZSTD_frameHeaderSize;
-    dctx->phase = 0;
-       dctx->previousDstEnd = NULL;
-       dctx->base = NULL;
+    ZSTD_resetDCtx(dctx);
     return dctx;
 }
 
@@ -1685,7 +1691,6 @@ size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx)
     return 0;
 }
 
-
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx)
 {
     return ((dctx_t*)dctx)->expected;
@@ -1697,8 +1702,8 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
 
     /* Sanity check */
     if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize;
-       if (dst != ctx->previousDstEnd)  /* not contiguous */
-               ctx->base = dst;
+    if (dst != ctx->previousDstEnd)  /* not contiguous */
+        ctx->base = dst;
 
     /* Decompress : frame header */
     if (ctx->phase == 0)
@@ -1754,7 +1759,7 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
         }
         ctx->phase = 1;
         ctx->expected = ZSTD_blockHeaderSize;
-               ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
+        ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
         return rSize;
     }
 

diff --git a/lib/zstd_static.h b/lib/zstd_static.h
index 1baa47d32162dddcc97a3a6b672fe4bc3e72c2c0..55a41012f55c101de11afa9769796a808f1beec4 100644 (file)
--- a/lib/zstd_static.h
+++ b/lib/zstd_static.h
@@ -56,6 +56,7 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
 
 typedef struct ZSTD_Dctx_s ZSTD_Dctx;
 ZSTD_Dctx* ZSTD_createDCtx(void);
+size_t     ZSTD_resetDCtx(ZSTD_Dctx* dctx);
 size_t     ZSTD_freeDCtx(ZSTD_Dctx* dctx);
 
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx);

diff --git a/programs/Makefile b/programs/Makefile
index 7046c087d3490aa5cfaab92ea1ae3bd5c7f696ee..981a12a7cd704ebc535617645af0c350377c50fa 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -120,10 +120,22 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32
 test-all: test test32 memtest
 
 test-zstd: zstd datagen
-       @echo "*** flush write error test ***"
+       @echo "\n**** frame concatenation **** "
+       @echo "hello " > hello.tmp
+       @echo "world!" > world.tmp
+       @cat hello.tmp world.tmp > helloworld.tmp
+       ./zstd hello.tmp > hello.zstd
+       ./zstd world.tmp > world.zstd
+       @cat hello.zstd world.zstd > helloworld.zstd
+       ./zstd -d helloworld.zstd > result.tmp
+       cat result.tmp
+       sdiff helloworld.tmp result.tmp
+       @rm *.tmp *.zstd
+       @echo frame concatenation test completed
+       @echo "**** flush write error test **** "
        echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
        echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
-       @echo "*** zstd round-trip tests *** "
+       @echo "**** zstd round-trip tests **** "
        ./datagen          | ./zstd -v    | ./zstd -d > $(VOID)
        ./datagen -g256MB  | ./zstd -v    | ./zstd -d > $(VOID)
        ./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)

diff --git a/programs/fileio.c b/programs/fileio.c
index 5b1ff06256cb3877373b411dd3a724ee692315d3..2da5632bb3b0a05f69cb2edd72f7362aa4f3b5fd 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -309,45 +309,18 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
 
 
 #define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
-unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
+unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
+                                       BYTE* inBuff, size_t inBuffSize,
+                                       BYTE* outBuff, size_t outBuffSize,
+                                       ZSTD_Dctx* dctx)
 {
-    FILE* finput, *foutput;
-    BYTE* inBuff;
-    size_t inBuffSize;
-    BYTE* outBuff, *op, *oend;
-    size_t outBuffSize;
-    U32   blockSize = 128 KB;
-    U32   wNbBlocks = 4;
+    BYTE* op = outBuff;
+    BYTE* const oend = outBuff + outBuffSize;
     U64   filesize = 0;
-    BYTE* header[MAXHEADERSIZE];
-    ZSTD_Dctx* dctx;
     size_t toRead;
     size_t sizeCheck;
 
 
-    /* Init */
-    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-    dctx = ZSTD_createDCtx();
-
-    /* check header */
-    toRead = ZSTD_nextSrcSizeToDecompress(dctx);
-    if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
-    sizeCheck = fread(header, (size_t)1, toRead, finput);
-    if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
-    sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
-    if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
-
-    /* Here later : blockSize determination */
-
-    /* Allocate Memory */
-    inBuffSize = blockSize + FIO_blockHeaderSize;
-    inBuff  = (BYTE*)malloc(inBuffSize);
-    outBuffSize = wNbBlocks * blockSize;
-    outBuff = (BYTE*)malloc(outBuffSize);
-    op = outBuff;
-    oend = outBuff + outBuffSize;
-    if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
-
     /* Main decompression Loop */
     toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     while (toRead)
@@ -380,6 +353,67 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
         toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     }
 
+    return filesize;
+}
+
+
+unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
+{
+    FILE* finput, *foutput;
+    BYTE* inBuff=NULL;
+    size_t inBuffSize = 0;
+    BYTE* outBuff=NULL;
+    size_t outBuffSize = 0;
+    U32   blockSize = 128 KB;
+    U32   wNbBlocks = 4;
+    U64   filesize = 0;
+    BYTE* header[MAXHEADERSIZE];
+    ZSTD_Dctx* dctx;
+    size_t toRead;
+    size_t sizeCheck;
+
+
+    /* Init */
+    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
+    dctx = ZSTD_createDCtx();
+
+    /* for each frame */
+    for ( ; ; )
+    {
+        /* check header */
+        ZSTD_resetDCtx(dctx);
+        toRead = ZSTD_nextSrcSizeToDecompress(dctx);
+        if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
+        sizeCheck = fread(header, (size_t)1, toRead, finput);
+        if (sizeCheck==0) break;   /* no more input */
+        if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
+        sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
+        if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
+
+        /* Here later : blockSize determination */
+
+        /* Allocate Memory (if needed) */
+        {
+            size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
+            size_t newOutBuffSize = wNbBlocks * blockSize;
+            if (newInBuffSize > inBuffSize)
+            {
+                free(inBuff);
+                inBuffSize = newInBuffSize;
+                inBuff  = (BYTE*)malloc(inBuffSize);
+            }
+            if (newOutBuffSize > outBuffSize)
+            {
+                free(outBuff);
+                outBuffSize = newOutBuffSize;
+                outBuff  = (BYTE*)malloc(outBuffSize);
+            }
+        }
+        if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
+
+        filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
+    }
+
     DISPLAYLEVEL(2, "\r%79s\r", "");
     DISPLAYLEVEL(2,"Decoded %llu bytes   \n", (long long unsigned)filesize);