From 32c9f715ae1532d4ea06ee5f811ed61c46e48d31 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 17 Oct 2017 14:07:43 -0700 Subject: [PATCH] fixed : Visual build compressing stdin with multi-threading enabled fails It was multiple reasons stacked : - Visual use a different code path, because ZSTD_NEWAPI is not defined - fileio.c sends `0` as `pledgedSrcSize` to mean `ZSTD_CONTENTSIZE_UNKNOWN` (fixed) - ZSTDMT_resetCCtx() interpreted `0` as "empty" instead of "unknown" (fixed) --- lib/compress/zstdmt_compress.c | 5 ++++- lib/compress/zstdmt_compress.h | 2 +- programs/Makefile | 4 ++-- programs/fileio.c | 6 +++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 197a2c49e..af7be3167 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -838,9 +838,12 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, /* ZSTDMT_resetCStream() : - * pledgedSrcSize is optional and can be zero == unknown */ + * pledgedSrcSize can be zero == unknown (for the time being) + * prefer using ZSTD_CONTENTSIZE_UNKNOWN, + * as `0` might mean "empty" in the future */ size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize) { + if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; if (zcs->params.nbThreads==1) return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize); return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, 0, zcs->params, diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 8c59c684f..a7ce7c7dd 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -50,7 +50,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, /* === Streaming functions === */ ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); -ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ +ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it may change in the future, to mean "empty" */ ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); diff --git a/programs/Makefile b/programs/Makefile index cbb67e9fe..1b21281b4 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -37,8 +37,8 @@ endif CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder \ - -DZSTD_NEWAPI \ - -DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library + -DXXH_NAMESPACE=ZSTD_ \ + -DZSTD_NEWAPI CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ diff --git a/programs/fileio.c b/programs/fileio.c index 17ad31b32..03e653893 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -817,11 +817,11 @@ static int FIO_compressFilename_internal(cRess_t ress, /* init */ #ifdef ZSTD_NEWAPI if (fileSize!=0) /* when src is stdin, fileSize==0, but is effectively unknown */ - ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); /* note : fileSize==0 means "empty" */ + ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); #elif defined(ZSTD_MULTITHREAD) - CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */ + CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize ? fileSize : ZSTD_CONTENTSIZE_UNKNOWN) ); #else - CHECK( ZSTD_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */ + CHECK( ZSTD_resetCStream(ress.cctx, fileSize ? fileSize : ZSTD_CONTENTSIZE_UNKNOWN) ); #endif /* Main compression loop */ -- 2.47.2