minor tests/Makefile refactoring

and update of zstd_manual,html

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 23224d77ab6da1a91d9e89c9d962d4f112886c01..02656c230f76f49080f6ed4b62c40f6aaec881a8 100644 (file)
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -34,7 +34,7 @@
   zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
   at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
   decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
-  Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
+  Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
   Compression can be done in:
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit memory management)
@@ -306,8 +306,8 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </pre></b><BR>
 <a name="Chapter12"></a><h2>Compressed size functions</h2><pre></pre>
 
-<pre><b>size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize);
-</b><p>  `src` should point to the start of a ZSTD encoded frame
+<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+</b><p>  `src` should point to the start of a ZSTD encoded frame or skippable frame
   `srcSize` must be at least as large as the frame
   @return : the compressed size of the frame pointed to by `src`, suitable to pass to
       `ZSTD_decompress` or similar, or an error code if given invalid input. 
@@ -321,7 +321,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
        to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
    @return : decompressed size of the frame pointed to be `src` if known, otherwise
              - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
-             - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) 
+             - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) 
 </p></pre><BR>
 
 <pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
@@ -365,7 +365,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </p></pre><BR>
 
 <pre><b>typedef enum {
-    ZSTD_p_forceWindow   </b>/* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/<b>
+    ZSTD_p_forceWindow   </b>/* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */<b>
 } ZSTD_CCtxParameter;
 </b></pre><BR>
 <pre><b>size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
@@ -585,6 +585,9 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
   c) Frame Content - any content (User Data) of length equal to Frame Size
   For skippable frames ZSTD_decompressContinue() always returns 0.
   For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+    Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
+           For purposes of decompression, it is valid in both cases to skip the frame using
+           ZSTD_findFrameCompressedSize to find its size in bytes.
   It also returns Frame Size as fparamsPtr->frameContentSize.
 <BR></pre>
 

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 97d2b38ee17bcd71eeacdcf78a9d7912036a7646..f32c334a9d1a6b21d58237d0406490c8fd107de9 100644 (file)
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -231,16 +231,16 @@ void ZSTDMT_compressChunk(void* jobDescription)
     const void* const src = (const char*)job->srcStart + job->dictSize;
     buffer_t const dstBuff = job->dstBuff;
     DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
-    if (job->cdict) {
+    if (job->cdict) {  /* should only happen for first segment */
         size_t const initError = ZSTD_compressBegin_usingCDict(job->cctx, job->cdict, job->fullFrameSize);
         if (job->cdict) DEBUGLOG(3, "using CDict ");
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
-    } else {
-        size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
+    } else {  /* srcStart points at reloaded section */
+        size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, 0);
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
         ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
     }
-    if (!job->firstChunk) {  /* flush frame header */
+    if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first segment */
         size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0);
         if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
         ZSTD_invalidateRepCodes(job->cctx);
@@ -248,7 +248,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
 
     DEBUGLOG(4, "Compressing : ");
     DEBUG_PRINTHEX(4, job->srcStart, 12);
-    job->cSize = (job->lastChunk) ?   /* last chunk signal */
+    job->cSize = (job->lastChunk) ?
                  ZSTD_compressEnd     (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
                  ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
     DEBUGLOG(3, "compressed %u bytes into %u bytes   (first:%u) (last:%u)", (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);

diff --git a/lib/zstd.h b/lib/zstd.h
index e597c5db5021dc660e02439bf992838f8dfc1f95..f35805447ff110e0a0ea5c4c09f9b05a2893a78d 100644 (file)
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -462,7 +462,7 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 
 typedef enum {
-    ZSTD_p_forceWindow   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/
+    ZSTD_p_forceWindow   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
 } ZSTD_CCtxParameter;
 /*! ZSTD_setCCtxParameter() :
  *  Set advanced parameters, selected through enum ZSTD_CCtxParameter

diff --git a/tests/.gitignore b/tests/.gitignore
index dc468dee4476e8a44feba21e218e492bb2277ba4..f408a7491213ffee071f438c9254e7d52aa7a014 100644 (file)
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -9,6 +9,8 @@ zbufftest32
 zbufftest-dll
 zstreamtest
 zstreamtest32
+zstreamtest_asan
+zstreamtest_tsan
 zstreamtest-dll
 datagen
 paramgrill

diff --git a/tests/Makefile b/tests/Makefile
index 5b0e29c672885d7c53c08ea66c8ba69a0b9b3c8a..4fae769d3895e83a2740a90d03222a346764bae4 100644 (file)
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -35,22 +35,28 @@ FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
-ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
+ZSTDCOMP_FILES   := $(ZSTDDIR)/compress/*.c
 ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
-ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+ZSTD_FILES  := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 
+ZSTD_OBJ  := $(patsubst %.c,%.o, $(wildcard $(ZSTD_FILES)) )
+ZBUFF_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZBUFF_FILES)) )
+ZDICT_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZDICT_FILES)) )
 
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
-MULTITHREAD = -DZSTD_MULTITHREAD
+MULTITHREAD_CPP = -DZSTD_MULTITHREAD
+MULTITHREAD_LD  =
 else
 EXT =
-MULTITHREAD = -pthread -DZSTD_MULTITHREAD
+MULTITHREAD_CPP = -DZSTD_MULTITHREAD
+MULTITHREAD_LD  = -lpthread
 endif
+MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)
 
 VOID = /dev/null
 ZSTREAM_TESTTIME = -T2mn
@@ -124,11 +130,23 @@ zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
        $(MAKE) -C $(ZSTDDIR) libzstd
        $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
 
-zstreamtest  : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
-       $(CC)      $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
+zstreamtest : CPPFLAGS += $(MULTITHREAD_CPP)
+zstreamtest : LDFLAGS += $(MULTITHREAD_LD)
+zstreamtest : $(ZSTREAMFILES)
+       $(CC) $(FLAGS) $^ -o $@$(EXT)
+
+zstreamtest32 : CFLAGS += -m32
+zstreamtest32 : $(ZSTREAMFILES)
+       $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+
+zstreamtest_asan : CFLAGS += -fsanitize=address
+zstreamtest_asan : $(ZSTREAMFILES)
+       $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
 
-zstreamtest32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
-       $(CC) -m32  $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+zstreamtest_tsan : CFLAGS += -fsanitize=thread
+zstreamtest_tsan : $(ZSTREAMFILES)
+       $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
 
 zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
 zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c