From: Stefano Lattarini <stefano.lattarini@gmail.com>
Date: Fri, 20 Jul 2012 18:53:57 +0000 (+0200)
Subject: [ng] coverage: recheck with many failed tests
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9761847dd0a49a14a46fa2ac09b7224e8e818147;p=thirdparty%2Fautomake.git

[ng] coverage: recheck with many failed tests

* t/parallel-tests-many.sh: Extend to check that the 'recheck' target
works even when a huge number of tests (~ 30k) have failed in the
previous testsuite run.  Currently this doesn't work, and causes the
test to fail, so list it ...
* Makefile.am (XFAIL_TESTS): ... in here.

Signed-off-by: Stefano Lattarini <stefano.lattarini@gmail.com>
---

diff --git a/Makefile.am b/Makefile.am
index 54de62fb9..290d5f423 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -350,6 +350,7 @@ XFAIL_TESTS = \
   t/interp3.sh \
   t/java-nobase.sh \
   t/objext-pr10128.sh \
+  t/parallel-tests-many.sh \
   t/pr8365-remake-timing.sh \
   t/remake-am-pr10111.sh \
   t/remake-m4-pr10111.sh \
diff --git a/t/parallel-tests-many.sh b/t/parallel-tests-many.sh
index 1932193ad..bd107e204 100755
--- a/t/parallel-tests-many.sh
+++ b/t/parallel-tests-many.sh
@@ -114,7 +114,15 @@ check_three_reruns ()
   test $(LC_ALL=C grep -c "^[A-Z][A-Z]*:" stdout) -eq 3
 }
 
-# We need to simulate a failure of few tests.
+$sleep
+touch $tst-1.test $dir1-1/foo.sh $dir2-1/$tst-1
+$MAKE check AM_LAZY_CHECK=yes > stdout || { cat stdout; exit 1; }
+cat stdout
+check_three_reruns
+grep "^# TOTAL: $whole_count$" stdout
+grep "^# PASS:  $whole_count$" stdout
+
+# We need to simulate the failure of few tests.
 st=0
 $MAKE check TESTS="$tst-1.test $dir1-1/foo.sh $dir2-1/$tst-1" \
             LOG_COMPILER=false > stdout && st=1
@@ -129,11 +137,32 @@ check_three_reruns
 grep "^# TOTAL: 3$" stdout
 grep "^# PASS:  3$" stdout
 
-$sleep
-touch $tst-1.test $dir1-1/foo.sh $dir2-1/$tst-1
-$MAKE check AM_LAZY_CHECK=yes > stdout || { cat stdout; exit 1; }
+# We need to simulate the failure of a lot of tests.
+$MAKE check LOG_COMPILER=false > stdout && { cat stdout; exit 1; }
 cat stdout
-check_three_reruns
+
+grep '^PASS:' stdout && exit 1
+# A random sample.
+grep "^FAIL: $tst-363\.test$" stdout
+grep "^FAIL: $dir1-9123/foo.sh$" stdout
+grep "^FAIL: $dir2-3609/$tst-3609$" stdout
+
+grep "^FAIL: " stdout > grp
+sed 20q grp # For debugging.
+test $(wc -l <grp) -eq $whole_count
+
+$MAKE recheck > stdout || { cat stdout; exit 1; }
+cat stdout
+
+grep '^FAIL:' stdout && exit 1
+# A random sample.
+grep "^PASS: $tst-363\.test$" stdout
+grep "^PASS: $dir1-9123/foo.sh$" stdout
+grep "^PASS: $dir2-3609/$tst-3609$" stdout
+
+grep "^PASS: " stdout > grp
+sed 20q grp # For debugging.
+test $(wc -l <grp) -eq $whole_count
 grep "^# TOTAL: $whole_count$" stdout
 grep "^# PASS:  $whole_count$" stdout