IO and bench : ZSTD_NEWAPI is the only remaining code path

removed the other 2 code paths (single thread, and ZSTDMT ones)
keeping only the new advanced API, for easier code coverage.

It shall also fix identified issue with Visual Studio
which doesn't have ZSTD_NEWAPI defined.

diff --git a/programs/Makefile b/programs/Makefile
index 1b21281b4828cf832ab4221f26da28111e6eac21..e1c5f50b6c1504da64f6050da1c9fb114b929c56 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -37,8 +37,7 @@ endif
 
 CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
            -I$(ZSTDDIR)/dictBuilder \
-           -DXXH_NAMESPACE=ZSTD_ \
-           -DZSTD_NEWAPI
+           -DXXH_NAMESPACE=ZSTD_
 CFLAGS  ?= -O3
 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \

diff --git a/programs/bench.c b/programs/bench.c
index efec1892aebde20107a455afc227417fb8a451f7..5b67d5c7629d2e32e8d9b1be9a4b93234c33b9fa 100644 (file)
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -287,7 +287,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                     U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
                     U32 nbLoops = 0;
                     ZSTD_CDict* cdict = NULL;
-#ifdef ZSTD_NEWAPI
                     ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
                     ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
                     ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
@@ -306,25 +305,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                     ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
                     ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
                     ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
-#else
-                    size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
-                    ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
-                    ZSTD_customMem const cmem = { NULL, NULL, NULL };
-                    if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
-                    if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
-                    if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
-                    if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
-                    if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
-                    if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
-                    if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
-                    cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem);
-                    if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
-#endif
                     do {
                         U32 blockNb;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                             size_t rSize;
-#ifdef ZSTD_NEWAPI
                             ZSTD_outBuffer out = { blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom, 0 };
                             ZSTD_inBuffer in = { blockTable[blockNb].srcPtr,  blockTable[blockNb].srcSize, 0 };
                             size_t cError = 1;
@@ -336,29 +320,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                                                 ZSTD_getErrorName(cError));
                             }
                             rSize = out.pos;
-#else  /* ! ZSTD_NEWAPI */
-                            if (dictBufferSize) {
-                                rSize = ZSTD_compress_usingCDict(ctx,
-                                                blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
-                                                blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
-                                                cdict);
-                            } else {
-#  ifdef ZSTD_MULTITHREAD       /* note : limitation : MT single-pass does not support compression with dictionary */
-                                rSize = ZSTDMT_compressCCtx(mtctx,
-                                                blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
-                                                blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
-                                                cLevel);
-#  else
-                                rSize = ZSTD_compress_advanced (ctx,
-                                                blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
-                                                blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
-                                                NULL, 0, zparams);
-#  endif
-                            }
-                            if (ZSTD_isError(rSize))
-                                EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s",
-                                            ZSTD_getErrorName(rSize));
-#endif  /* ZSTD_NEWAPI */
                             blockTable[blockNb].cSize = rSize;
                         }
                         nbLoops++;

diff --git a/programs/fileio.c b/programs/fileio.c
index 4ddb8f8e010a3c222e734bc94f3697b1088b190f..c8eab06791e8a001b85d10dec9b96235c8f22ca5 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -393,11 +393,7 @@ typedef struct {
     size_t srcBufferSize;
     void*  dstBuffer;
     size_t dstBufferSize;
-#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
-    ZSTDMT_CCtx* cctx;
-#else
     ZSTD_CStream* cctx;
-#endif
 } cRess_t;
 
 static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
@@ -406,24 +402,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
     cRess_t ress;
     memset(&ress, 0, sizeof(ress));
 
-#ifdef ZSTD_NEWAPI
     ress.cctx = ZSTD_createCCtx();
     if (ress.cctx == NULL)
         EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
-#elif defined(ZSTD_MULTITHREAD)
-    ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
-    if (ress.cctx == NULL)
-        EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx");
-    if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
-        /* use complete window for overlap */
-        ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);
-    if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET)
-        ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog);
-#else
-    ress.cctx = ZSTD_createCStream();
-    if (ress.cctx == NULL)
-        EXM_THROW(30, "allocation error : can't create ZSTD_CStream");
-#endif
     ress.srcBufferSize = ZSTD_CStreamInSize();
     ress.srcBuffer = malloc(ress.srcBufferSize);
     ress.dstBufferSize = ZSTD_CStreamOutSize();
@@ -437,68 +418,38 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
         if (dictFileName && (dictBuffer==NULL))
             EXM_THROW(32, "allocation error : can't create dictBuffer");
 
-#ifdef ZSTD_NEWAPI
-        {   /* frame parameters */
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
-            /* compression level */
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
-            /* long distance matching */
-            CHECK( ZSTD_CCtx_setParameter(
-                          ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
-            if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
-                CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
-            }
-            if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
-                CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
-            }
-            /* compression parameters */
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
-            /* multi-threading */
-            DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
-            /* dictionary */
-            CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );  /* just to load dictionary with good compression parameters */
-            CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
-            CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) );  /* reset */
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );  /* always enable content size, when available (note: supposed to be default anyway) */
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
+        /* compression level */
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
+        /* long distance matching */
+        CHECK( ZSTD_CCtx_setParameter(
+                      ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
+        if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
         }
-#elif defined(ZSTD_MULTITHREAD)
-        {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
-            params.fParams.checksumFlag = g_checksumFlag;
-            params.fParams.noDictIDFlag = !g_dictIDFlag;
-            if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
-            if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
-            if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
-            if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
-            if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
-            if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
-            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
-            CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
-            ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
+        if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
         }
-#else
-        {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
-            params.fParams.checksumFlag = g_checksumFlag;
-            params.fParams.noDictIDFlag = !g_dictIDFlag;
-            if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
-            if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
-            if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
-            if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
-            if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
-            if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
-            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
-            CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
-        }
-#endif
+        /* compression parameters */
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
+        /* multi-threading */
+        DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
+        /* dictionary */
+        CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );  /* just to load dictionary with good compression parameters */
+        CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
+        CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) );  /* reset */
+
         free(dictBuffer);
     }
 
@@ -509,11 +460,7 @@ static void FIO_freeCResources(cRess_t ress)
 {
     free(ress.srcBuffer);
     free(ress.dstBuffer);
-#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
-    ZSTDMT_freeCCtx(ress.cctx);
-#else
     ZSTD_freeCStream(ress.cctx);   /* never fails */
-#endif
 }
 
 
@@ -814,14 +761,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
     }
 
     /* init */
-#ifdef ZSTD_NEWAPI
-    if (fileSize!=UTIL_FILESIZE_UNKNOWN)  /* when src is stdin, fileSize==0, but is effectively unknown */
+    if (fileSize != UTIL_FILESIZE_UNKNOWN)
         ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
-#elif defined(ZSTD_MULTITHREAD)
-    CHECK( ZSTDMT_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) );
-#else
-    CHECK( ZSTD_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) );
-#endif
 
     /* Main compression loop */
     while (1) {
@@ -833,14 +774,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
 
         while (inBuff.pos != inBuff.size) {
             ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
-#ifdef ZSTD_NEWAPI
             CHECK( ZSTD_compress_generic(ress.cctx,
                         &outBuff, &inBuff, ZSTD_e_continue) );
-#elif defined(ZSTD_MULTITHREAD)
-            CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) );
-#else
-            CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) );
-#endif
 
             /* Write compressed stream */
             if (outBuff.pos) {
@@ -871,21 +806,15 @@ static int FIO_compressFilename_internal(cRess_t ress,
     {   size_t result = 1;
         while (result != 0) {
             ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
-#ifdef ZSTD_NEWAPI
             ZSTD_inBuffer inBuff = { NULL, 0, 0 };
             result = ZSTD_compress_generic(ress.cctx,
                         &outBuff, &inBuff, ZSTD_e_end);
-#elif defined(ZSTD_MULTITHREAD)
-            result = ZSTDMT_endStream(ress.cctx, &outBuff);
-#else
-            result = ZSTD_endStream(ress.cctx, &outBuff);
-#endif
             if (ZSTD_isError(result)) {
                 EXM_THROW(26, "Compression error during frame end : %s",
                             ZSTD_getErrorName(result));
             }
             {   size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
-                if (sizeCheck!=outBuff.pos)
+                if (sizeCheck != outBuff.pos)
                     EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName);
             }
             compressedfilesize += outBuff.pos;