extended static dictionary to buffered mode

diff --git a/NEWS b/NEWS
index 50b26fb95c95fa7cae22e60040d27f4303d4fc22..939c1ef9a6e631aeff5488c70bce863481c61326 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
 v0.4.4
 Fixed : high compression modes for Windows 32 bits
+new : external dictionary API extended to buffered mode
 new : windows DLL project, thanks to Christophe Chevalier
 
 v0.4.3 :

diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c
index 844c50405a5424e729c0298c9065edace634e4ef..c726f0de003a6126faf4257fbb46f66d33d5f534 100644 (file)
--- a/lib/zstd_buffered.c
+++ b/lib/zstd_buffered.c
@@ -160,6 +160,12 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
 }
 
 
+ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* zbc, const void* src, size_t srcSize)
+{
+    ZSTD_compress_insertDictionary(zbc->zc, src, srcSize);
+    return 0;
+}
+
 
 /* *** Compression *** */
 
@@ -327,6 +333,8 @@ struct ZBUFF_DCtx_s {
     size_t outStart;
     size_t outEnd;
     size_t hPos;
+    const char* dict;
+    size_t dictSize;
     ZBUFF_dStage stage;
     unsigned char headerBuffer[ZSTD_frameHeaderSize_max];
 };   /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
@@ -353,17 +361,23 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
 }
 
 
-
 /* *** Initialization *** */
 
 size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
 {
     zbc->stage = ZBUFFds_readHeader;
-    zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0;
+    zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = zbc->dictSize = 0;
     return ZSTD_resetDCtx(zbc->zc);
 }
 
 
+size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* zbc, const void* src, size_t srcSize)
+{
+    zbc->dict = src;
+    zbc->dictSize = srcSize;
+    return 0;
+}
+
 
 /* *** Decompression *** */
 
@@ -442,6 +456,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
                         if (zbc->outBuff == NULL) return ERROR(memory_allocation);
                     }
                 }
+                if (zbc->dictSize)
+                    ZSTD_decompress_insertDictionary(zbc->zc, zbc->dict, zbc->dictSize);
                 if (zbc->hPos)
                 {
                     /* some data already loaded into headerBuffer : transfer into inBuff */

diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h
index 755728b1b363709e8edfc368f850f1672f213ad7..d2316a82b7bb128f3d96b4ca036825d98049a3bd 100644 (file)
--- a/lib/zstd_buffered.h
+++ b/lib/zstd_buffered.h
@@ -69,6 +69,7 @@ ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void);
 ZSTDLIB_API size_t      ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
 
 ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
+ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* cctx, const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
 ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
 ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
@@ -81,6 +82,9 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDst
 *  Use ZBUFF_compressInit() to start a new compression operation.
 *  ZBUFF_CCtx objects can be reused multiple times.
 *
+*  Optionally, a reference to a static dictionary can be created with ZBUFF_compressWithDictionary()
+*  Note that the dictionary content must remain accessible during the compression process.
+*
 *  Use ZBUFF_compressContinue() repetitively to consume input stream.
 *  *srcSizePtr and *maxDstSizePtr can be any size.
 *  The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr.
@@ -115,6 +119,8 @@ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void);
 ZSTDLIB_API size_t      ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
 
 ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
+ZSTDLIB_API size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* dctx, const void* src, size_t srcSize);
+
 ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
 
 /** ************************************************
@@ -125,6 +131,10 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t*
 *  Use ZBUFF_decompressInit() to start a new decompression operation.
 *  ZBUFF_DCtx objects can be reused multiple times.
 *
+*  Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary()
+*  It must be the same content as the one set during compression phase.
+*  Dictionary content must remain accessible during the decompression process.
+*
 *  Use ZBUFF_decompressContinue() repetitively to consume your input.
 *  *srcSizePtr and *maxDstSizePtr can be any size.
 *  The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.

diff --git a/programs/zbufftest.c b/programs/zbufftest.c
index 590393561de6db05f76882dc4d213da6ab0e0fe5..ab8aa3486e6d2af3c45f622eeab508be6687d9d8 100644 (file)
--- a/programs/zbufftest.c
+++ b/programs/zbufftest.c
@@ -161,6 +161,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     ZBUFF_compressInit(zc, 1);
     readSize = CNBufferSize;
     genSize = compressedBufferSize;
+    ZBUFF_compressWithDictionary(zc, CNBuffer, 128 KB);
     result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
     if (ZBUFF_isError(result)) goto _output_error;
     if (readSize != CNBufferSize) goto _output_error;   /* entire input should be consumed */
@@ -174,6 +175,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     /* Basic decompression test */
     DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
     ZBUFF_decompressInit(zd);
+    ZBUFF_decompressWithDictionary(zd, CNBuffer, 128 KB);
     readSize = cSize;
     genSize = CNBufferSize;
     result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
@@ -244,7 +246,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
     U32 coreSeed = seed, lseed = 0;
     ZBUFF_CCtx* zc;
     ZBUFF_DCtx* zd;
-    XXH64_state_t crc64;
     U32 startTime = FUZ_GetMilliStart();
 
     /* allocation */
@@ -280,10 +281,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
     for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
     {
         size_t sampleSize, sampleStart;
-        size_t cSize;
+        const BYTE* dict;
+        size_t cSize, dictSize;
         size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
         size_t errorCode;
         U32 sampleSizeLog, buffNb, n, nbChunks;
+        XXH64_state_t crc64;
         U64 crcOrig, crcDest;
 
         /* init */
@@ -316,6 +319,15 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
         maxTestSize = (size_t)1 << sampleSizeLog;
         maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
         ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
+
+        sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
+        sampleSize = (size_t)1 << sampleSizeLog;
+        sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+        sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+        dict = srcBuffer + sampleStart;
+        dictSize = sampleSize;
+        ZBUFF_compressWithDictionary(zc, dict, dictSize);
+
         totalTestSize = 0;
         cSize = 0;
         for (n=0; n<nbChunks; n++)
@@ -363,6 +375,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
 
         /* multi - fragments decompression test */
         ZBUFF_decompressInit(zd);
+        ZBUFF_decompressWithDictionary(zd, dict, dictSize);
         totalCSize = 0;
         totalGenSize = 0;
         while (totalCSize < cSize)