From: Yann Collet Date: Thu, 28 Sep 2017 00:27:38 +0000 (-0700) Subject: reduced zstreamtest --mt memory load X-Git-Tag: v1.3.2~3^2~21^2~15 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bc32b40b9808f8b6c2d4526ae9738f5240ac01f4;p=thirdparty%2Fzstd.git reduced zstreamtest --mt memory load adjust compression level, hence memory usage, depending on nb threads in order to run correctly on memory-starved VM. --- diff --git a/Makefile b/Makefile index 5b3f5fdf7..ac53c2dd8 100644 --- a/Makefile +++ b/Makefile @@ -296,7 +296,7 @@ endif #------------------------------------------------------------------------ -#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +# target specific tests #------------------------------------------------------------------------ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) cmakebuild: @@ -306,38 +306,38 @@ cmakebuild: cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall c90build: clean - gcc -v + $(CC) -v CFLAGS="-std=c90" $(MAKE) allmost # will fail, due to missing support for `long long` gnu90build: clean - gcc -v + $(CC) -v CFLAGS="-std=gnu90" $(MAKE) allmost c99build: clean - gcc -v + $(CC) -v CFLAGS="-std=c99" $(MAKE) allmost gnu99build: clean - gcc -v + $(CC) -v CFLAGS="-std=gnu99" $(MAKE) allmost c11build: clean - gcc -v + $(CC) -v CFLAGS="-std=c11" $(MAKE) allmost bmix64build: clean - gcc -v + $(CC) -v CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test bmix32build: clean - gcc -v + $(CC) -v CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test bmi32build: clean - gcc -v + $(CC) -v CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test staticAnalyze: clean - gcc -v + $(CC) -v CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all endif diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 3f190f050..28259dda8 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -741,20 +741,20 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres static const U32 maxSampleLog = 19; size_t const srcBufferSize = (size_t)1<= srcBufferSize) - maxTestSize = srcBufferSize-1; + maxTestSize = MIN(maxTestSize, srcBufferSize-16); { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) ); } @@ -999,15 +998,16 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp U32 result = 0; U32 testNb = 0; U32 coreSeed = seed; - ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(2); /* will be reset sometimes */ + U32 nbThreads = 2; + ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads); /* will be reset sometimes */ ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ ZSTD_DStream* const zd_noise = ZSTD_createDStream(); clock_t const startClock = clock(); const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ size_t dictSize = 0; U32 oldTestLog = 0; - U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests; - U32 const nbThreadsMax = bigTests ? 5 : 2; + int const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests; + U32 const nbThreadsMax = bigTests ? 4 : 2; /* allocations */ cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); @@ -1043,8 +1043,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp size_t maxTestSize; /* init */ - if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } - else { DISPLAYUPDATE(2, "\r%6u ", testNb); } + if (testNb < nbTests) { + DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); + } else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); lseed = coreSeed ^ prime32; @@ -1052,7 +1053,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp /* some issues can only happen when reusing states */ if ((FUZ_rand(&lseed) & 0xFF) == 131) { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) % 6) + 1; - U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax); + nbThreads = MIN(nbThreadsCandidate, nbThreadsMax); DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads); ZSTDMT_freeCCtx(zc); zc = ZSTDMT_createCCtx(nbThreads); @@ -1092,11 +1093,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp } else { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const cLevelCandidate = (FUZ_rand(&lseed) % - (ZSTD_maxCLevel() - - (MAX(testLog, dictLog) / 3))) + - 1; - U32 const cLevel = MIN(cLevelCandidate, cLevelMax); + int const cLevelCandidate = ( FUZ_rand(&lseed) + % (ZSTD_maxCLevel() - (MAX(testLog, dictLog) / 2)) ) + + 1; + int const cLevelThreadAdjusted = cLevelCandidate - (nbThreads * 2) + 2; /* reduce cLevel when multiple threads to reduce memory consumption */ + int const cLevelMin = MAX(cLevelThreadAdjusted, 1); /* no negative cLevel yet */ + int const cLevel = MIN(cLevelMin, cLevelMax); maxTestSize = FUZ_rLogLength(&lseed, testLog); oldTestLog = testLog; /* random dictionary selection */