From 97540c57f3fbe7c2d72eb729162f8f9cbe81903d Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Thu, 19 Jul 2012 13:42:19 +0200 Subject: [PATCH] [ng] coverage: testing with lots of test scripts See long-standing automake bug#7868. * t/parallel-tests-many.sh: Simplify and enhance. Among other things, this test now tries running ~ 30k tests. Currently fails on several systems (e.g., Linux 2.6.30 on i686, Solaris 10 on i86pc). Signed-off-by: Stefano Lattarini --- t/parallel-tests-many.sh | 240 ++++++++++++++++----------------------- 1 file changed, 99 insertions(+), 141 deletions(-) diff --git a/t/parallel-tests-many.sh b/t/parallel-tests-many.sh index 0b4fef2bf..1932193ad 100755 --- a/t/parallel-tests-many.sh +++ b/t/parallel-tests-many.sh @@ -16,175 +16,133 @@ # Check that the parallel testsuite harness does not hit errors due # to an exceeded command line length when there are many tests. -# For automake bug#7868. This test is currently expected to fail. +# For automake bug#7868. . ./defs || exit 1 -cat >> configure.ac << 'END' -AC_OUTPUT -END +echo AC_OUTPUT >> configure.ac -cat > Makefile.am <<'END' -# Sanity check that the $(TESTS) is going to exceed the system -# command line length. -# Extra quoting and indirections below are required to ensure the -# various make implementations (e.g, GNU make or Sun Distributed Make) -# will truly spawn a shell to execute this command, instead of relying -# on optimizations that might mask the "Argument list too long" error -# we expect. -this-will-fail: - @":" && ":" $(TEST_LOGS) -TEST_LOG_COMPILER = true -include list-of-tests.am -# So that we won't have to create a ton of dummy test cases. -$(TESTS): +cat > Makefile.am << 'END' +TEST_EXTENSIONS = .test .sh +LOG_COMPILER = true +TEST_LOG_COMPILER = $(LOG_COMPILER) +SH_LOG_COMPILER = $(LOG_COMPILER) +EXTRA_DIST = $(TESTS) END -# The real instance will be dynamically created later. -echo TESTS = foo.test > list-of-tests.am - -$ACLOCAL && $AUTOCONF && $AUTOMAKE -a \ - || framework_failure_ "unexpected autotools failure" -./configure \ - || framework_failure_ "unexpected configure failure" - -# We want to hit the system command-line length limit without hitting -# the filename length limit or the PATHMAX limit; so we use longish -# (but not too long) names for the testcase, and place them in a nested -# (but not too deeply) directory. -# We also prefer to use the minimal(ish) number of test cases that can -# make us hit the command-line length limit, since the more the test -# cases are, the more time "automake" and "make check" will take to run -# (especially on Cygwin and MinGW/MSYS). - -tname="wow-this-is-a-very-long-name-for-a-simple-dummy-test-case" -dname="and-this-too-is-a-very-long-name-for-a-dummy-directory" - -deepdir=. -depth=0 -for i in 1 2 3 4 5 6 7 8 9 10 12 13 14 15 16 17 18 19 29 21 22 23 24; do - new_deepdir=$deepdir/$dname.d$i - mkdir $new_deepdir || break - tmpfile=$new_deepdir/$tname-some-more-chars-for-good-measure - if touch $tmpfile; then - rm -f $tmpfile || exit 99 - else - rmdir $new_deepdir || exit 99 - fi - deepdir=$new_deepdir - unset tmpfile new_deepdir - depth=$i -done - -cat < "all"; - use strict; - print "TESTS = \\\n"; - my $i = 0; - while (++$i) - { - print " $ENV{deepdir}/$ENV{tname}-$i.test"; - if ($i >= $ENV{count}) - { - print "\n"; - last; - } - else - { - print " \\\n"; - } - } - ' > list-of-tests.am || exit 99 - sed 20q list-of-tests.am || exit 99 # For debugging. - $AUTOMAKE Makefile \ - || framework_failure_ "unexpected automake failure" - ./config.status Makefile \ - || framework_failure_ "unexpected config.status failure" + find . -name '*.log' | $EGREP -v '^\./(config|test-suite)\.log$' } -for count in 1 2 4 8 12 16 20 24 28 32 48 64 96 128 E_HUGE; do - test $count = E_HUGE && break - count=$(($count * 100)) - setup_data - if $MAKE this-will-fail; then - continue - else - # We have managed to find a number of test cases large enough to - # hit the system command-line limits; we can stop. But first, for - # good measure, increase the number of tests of some 20%, to be - # "even more sure" of really tickling command line length limits. - count=$(($count * 12)) - count=$(($count / 10)) - setup_data - break - fi -done - -if test $count = E_HUGE; then - framework_failure_ "system has a too-high limit on command line length" -else - cat < $f + echo $f + done + i=$(($i + 1)) + # Disable shell traces after the first iteration, to avoid + # polluting the test logs. + set +x +done > t +set -x # Re-enable shell traces. +echo 'TESTS = \' >> Makefile.am +sed '$!s/$/ \\/' t >> Makefile.am +rm -f t + +whole_count=$(($count * 3)) + +test $(wc -l stdout || { cat stdout; exit 1; } cat stdout -grep "^# TOTAL: $count$" stdout -grep "^# PASS: $count$" stdout +grep "^# TOTAL: $whole_count$" stdout +grep "^# PASS: $whole_count$" stdout + +# Only check head, tail, and a random sample. + +test -f $tst-1.log +test -f $dir1-1/foo.log +test -f $dir2-1/$tst-1.log + +test -f $tst-$count.log +test -f $dir1-$count/foo.log +test -f $dir2-$count/$tst-$count.log -grep "^PASS: .*$tname-[0-9][0-9]*\.test" stdout > grp -ls -1 $deepdir | grep '\.log$' > lst +test -f $tst-163.log +grep "^PASS: $tst-163\.test$" stdout +test -f $dir1-7645/foo.log +grep "^PASS: $dir1-7645/foo.sh$" stdout +test -f $dir2-4077/$tst-4077.log +grep "^PASS: $dir2-4077/$tst-4077$" stdout + +grep "^PASS: " stdout > grp +list_logs > lst sed 20q lst # For debugging. sed 20q grp # Likewise. -test $(cat stdout && st=1 +$MAKE check TESTS="$tst-1.test $dir1-1/foo.sh $dir2-1/$tst-1" \ + LOG_COMPILER=false > stdout && st=1 cat stdout -test $(grep -c '^FAIL:' stdout) -eq 2 || st=1 -test $st -eq 0 || fatal_ "couldn't simulate failure of two tests" +test $(grep -c '^FAIL:' stdout) -eq 3 || st=1 +test $st -eq 0 || fatal_ "couldn't simulate failure of 3 tests" unset st $MAKE recheck > stdout || { cat stdout; exit 1; } cat stdout -grep "^PASS: .*$tname-1\.test" stdout -grep "^PASS: .*$tname-2\.test" stdout -test $(LC_ALL=C grep -c "^[A-Z][A-Z]*:" stdout) -eq 2 -grep "^# TOTAL: 2$" stdout -grep "^# PASS: 2$" stdout - -# "make clean" might ignore some failures, so we prefer to also grep its -# output to ensure that no "Argument list too long" error was encountered. +check_three_reruns +grep "^# TOTAL: 3$" stdout +grep "^# PASS: 3$" stdout + +$sleep +touch $tst-1.test $dir1-1/foo.sh $dir2-1/$tst-1 +$MAKE check AM_LAZY_CHECK=yes > stdout || { cat stdout; exit 1; } +cat stdout +check_three_reruns +grep "^# TOTAL: $whole_count$" stdout +grep "^# PASS: $whole_count$" stdout + +# "make clean" might ignore some failures (either on purpose or spuriously), +# so we prefer to also grep its output to ensure that no "Argument list too +# long" error was encountered. $MAKE clean >output 2>&1 || { cat output; exit 1; } cat output grep -i 'list.* too long' output && exit 1 -ls $deepdir | grep '\.log$' && exit 1 +list_logs | grep . && exit 1 : -- 2.47.2