From: Sen Huang <senhuang96@fb.com>
Date: Thu, 9 Sep 2021 15:55:43 +0000 (-0700)
Subject: Add a dictionary training large corpus test
X-Git-Tag: v1.5.1~1^2~114^2~1
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4a498fb9c35fa61a0c07f873fb891e11b92804a1;p=thirdparty%2Fzstd.git

Add a dictionary training large corpus test
---

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 774655aa9..45d1d3fe8 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -936,8 +936,13 @@ cat tmp | zstd -14 -f --size-hint=5500  | zstd -t  # considerably too low
 
 
 println "\n===>  dictionary tests "
-
-println "- test with raw dict (content only) "
+println "- Test high/low compressibility corpus training"
+datagen -g12M -P90 > tmpCorpusHighCompress
+datagen -g12M -P5 > tmpCorpusLowCompress
+zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress
+zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress
+rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress
+println "- Test with raw dict (content only) "
 datagen > tmpDict
 datagen -g1M | $MD5SUM > tmp1
 datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2