* tests/defs (count_test_results): New function.
* tests/check11.test: Use it.
* tests/test-driver-custom-multitest.test: Likewise.
* tests/test-driver-custom-multitest-recheck.test: Likewise.
* tests/test-driver-custom-multitest-recheck2.test: Likewise.
* tests/parallel-tests-log-override-recheck.test: Likewise.
* tests/parallel-tests-log-override-recheck.test: Likewise.
* tests/parallel-tests-no-spurious-summary.test: Likewise, and
slightly improve debugging output.
* tests/parallel-tests.test: Make use of `count_test_results'.
Also, make grepping of "make check" output slightly stricter
* tests/parallel-tests9.test: Likewise.
* tests/parallel-tests-log-override-2.test: Likewise, and throw
in a small optimization.
+2011-07-07 Stefano Lattarini <stefano.lattarini@gmail.com>
+
+ test defs: new auxiliary function 'count_test_results'
+ * tests/defs (count_test_results): New function.
+ * tests/check11.test: Use it.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-no-spurious-summary.test: Likewise, and
+ slightly improve debugging output.
+ * tests/parallel-tests.test: Make use of `count_test_results'.
+ Also, make grepping of "make check" output slightly stricter
+ * tests/parallel-tests9.test: Likewise.
+ * tests/parallel-tests-log-override-2.test: Likewise, and throw
+ in a small optimization.
+
2011-07-07 Stefano Lattarini <stefano.lattarini@gmail.com>
parallel-tests: simplify testsuite summary
$AUTOMAKE -a
./configure
+
env TESTS=skip $MAKE -e check >stdout
cat stdout
-grep '1.*passed' stdout && Exit 1
+if test x"$parallel_tests" = x"yes"; then
+ count_test_results total=1 pass=0 fail=0 skip=1 xfail=0 xpass=0 error=0
+else
+ grep '1.*passed' stdout && Exit 1
+ : For shells with buggy 'set -e'.
+fi
env TESTS="skip skip2" $MAKE -e check >stdout
cat stdout
-grep '2.*passed' stdout && Exit 1
+if test x"$parallel_tests" = x"yes"; then
+ count_test_results total=2 pass=0 fail=0 skip=2 xfail=0 xpass=0 error=0
+else
+ grep '2.*passed' stdout && Exit 1
+ : For shells with buggy 'set -e'.
+fi
:
esac
}
+# count_test_results total=N pass=N fail=N xpass=N xfail=N skip=N error=N
+# -----------------------------------------------------------------------
+# Check that a testsuite run driven by the parallel-tests harness has
+# had the specified numbers of test results (specified by kind).
+# This function assumes that the output of "make check" or "make recheck"
+# has been saved in the `stdout' file in the current directory, and its
+# log in the `test-suite.log' file.
+count_test_results ()
+{
+ # Use a subshell so that we won't pollute the script namespace.
+ (
+ # TODO: Do proper checks on the arguments?
+ total=ERR pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
+ eval "$@"
+ # For debugging.
+ $EGREP -i '(total|x?pass|x?fail|skip|error)' stdout || :
+ rc=0
+ # Avoid spurious failures with shells with "overly sensible"
+ # `errexit' shell flag, such as e.g., Solaris /bin/sh.
+ set +e
+ test `grep -c '^PASS:' stdout` -eq $pass || rc=1
+ test `grep -c '^XFAIL:' stdout` -eq $xfail || rc=1
+ test `grep -c '^SKIP:' stdout` -eq $skip || rc=1
+ test `grep -c '^FAIL:' stdout` -eq $fail || rc=1
+ test `grep -c '^XPASS:' stdout` -eq $xpass || rc=1
+ test `grep -c '^ERROR:' stdout` -eq $error || rc=1
+ grep "^# TOTAL: *$total$" stdout || rc=1
+ grep "^# PASS: *$pass$" stdout || rc=1
+ grep "^# XFAIL: *$xfail$" stdout || rc=1
+ grep "^# SKIP: *$skip$" stdout || rc=1
+ grep "^# FAIL: *$fail$" stdout || rc=1
+ grep "^# XPASS: *$xpass$" stdout || rc=1
+ grep "^# ERROR: *$error$" stdout || rc=1
+ test $rc -eq 0
+ )
+}
+
commented_sed_unindent_prog='
/^$/b # Nothing to do for empty lines.
x # Get x<indent> into pattern space.
$MAKE -e check >stdout || { cat stdout; Exit 1; }
cat stdout
ls -l
+ count_test_results total=2 pass=1 fail=0 skip=1 xfail=0 xpass=0 error=0
cat pass.log
cat skip.log
cat partial.log
test ! -f pass2.log
test ! -f skip2.log
test ! -f fail.log
- grep '^PASS: .*pass\.test' stdout
- grep '^SKIP: .*skip\.test' stdout
+ grep '^PASS: pass\.test$' stdout
+ grep '^SKIP: skip\.test$' stdout
$FGREP 'skip.test' partial.log
$FGREP '% skipped test %' partial.log
- for t in pass2 skip2 fail; do
- $FGREP "$t.test" stdout && Exit 1
- $FGREP "$t.test" partial.log && Exit 1
- done
+ $EGREP '(pass2|skip2|fail)\.test' stdout partial.log && Exit 1
rm -f *.log
done
&& { cat stdout; Exit 1; }
cat stdout
ls -l
-grep '^# TOTAL: *2$' stdout
-grep '^# FAIL: *1$' stdout
-grep '^# ERROR: *1$' stdout
+count_test_results total=2 pass=0 fail=1 skip=0 xfail=0 xpass=0 error=1
for x in stdout my.log; do
$FGREP foo.test $x && Exit 1
$FGREP bar.test $x
&& { cat stdout; Exit 1; }
cat stdout
ls -l
-grep '^# TOTAL: *2$' stdout
-grep '^# PASS: *1$' stdout
-grep '^# ERROR: *1$' stdout
+count_test_results total=2 pass=1 fail=0 skip=0 xfail=0 xpass=0 error=1
$FGREP foo.test stdout && Exit 1
$FGREP bar.test stdout
$FGREP baz.test stdout
./configure
st=0
-$MAKE check >output 2>&1 || st=$?
-cat output
+$MAKE check >stdout || st=$?
+cat stdout
cat test-suite.log
cat foo.log
+cat bar.log
test $st -eq 0 || Exit $st
grep '^:test-result:XFAIL$' foo.log
grep '^:test-result:FAIL$' bar.log
grep '^:test-result: XPASS$' bar.log
-grep '^# TOTAL: *2$' output
-grep '^# PASS: *2$' output
-for result in FAIL XFAIL XPASS SKIP ERROR; do
- grep "^# $result: *0$" output
-done
+count_test_results total=2 pass=2 fail=0 skip=0 xfail=0 xpass=0 error=0
:
$MAKE check >stdout && { cat stdout; Exit 1; }
cat stdout
-# There should be one failure and one hard error.
-test `grep -c '^FAIL:' stdout` -eq 1
-test `grep -c '^ERROR:' stdout` -eq 1
+count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
test -f mylog.log
cat mylog.log
test `grep -c '^FAIL:' mylog.log` -eq 1
test `grep -c '^ERROR:' mylog.log` -eq 1
+$EGREP '^(X?PASS|XFAIL|SKIP)' mylog.log && Exit 1
test -f baz.log
test -f bar.log
test -f foo.log
# Note that this usage has a problem: the summary will only
# take bar.log into account, because the $(TEST_SUITE_LOG) rule
# does not "see" baz.log. Hmm.
-env TESTS='bar.test' $MAKE -e check && Exit 1
+env TESTS='bar.test' $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^FAIL: baz\.test$' stdout
+grep '^ERROR: bar\.test$' stdout
+
test -f baz.log
test -f bar.log
test ! -f foo.log
env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
cat stdout
test -f foo.log
-grep foo.test stdout
+grep '^PASS: foo\.test$' stdout
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
grep '^# PASS: *1$' stdout
./configure
$MAKE check >stdout && { cat stdout; Exit 1; }
cat stdout
-grep '^# PASS: *1$' stdout
-grep '^# FAIL: *1$' stdout
-grep '^# ERROR: *1$' stdout
+count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
$MAKE recheck >stdout && { cat stdout; Exit 1; }
cat stdout
-grep foo.test stdout && Exit 1
-grep bar.test stdout || Exit 1
-grep baz.test stdout || Exit 1
-grep '^# PASS: *0$' stdout
-grep '^# FAIL: *1$' stdout
-grep '^# ERROR: *1$' stdout
+count_test_results total=2 pass=0 fail=1 skip=0 xfail=0 xpass=0 error=1
+grep 'foo\.test' stdout && Exit 1
+grep '^ERROR: bar\.test$' stdout
+grep '^FAIL: baz\.test$' stdout
# If we cannot read the log file, then redo it as well.
chmod a-r foo.log
if test ! -r foo.log; then
$MAKE recheck >stdout && { cat stdout; Exit 1; }
cat stdout
- grep foo.test stdout || Exit 1
+ count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
+ grep '^PASS: foo\.test$' stdout || Exit 1
fi
# Ensure that recheck builds check_SCRIPTS, and that
cat stdout; ls -l
}
-do_count ()
-{
- pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
- eval "$@"
- $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP|ERROR)' stdout || : # For debugging.
- test `grep -c '^PASS:' stdout` -eq $pass
- test `grep -c '^FAIL:' stdout` -eq $fail
- test `grep -c '^XPASS:' stdout` -eq $xpass
- test `grep -c '^XFAIL:' stdout` -eq $xfail
- test `grep -c '^SKIP:' stdout` -eq $skip
- test `grep -c '^ERROR:' stdout` -eq $error
-}
-
for vpath in : false; do
if $vpath; then
mkdir build
test ! -r c.log
test ! -r d.run
test ! -r d.log
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
: Run the tests for the first time.
$MAKE check >stdout && { cat stdout; Exit 1; }
test -f b.run
test -f c.run
test -f d.run
- do_count pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
+ count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
: Let us make b.test pass.
echo OK > b.ok
test -f b.run
test -f c.run
test -f d.run
- do_count pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
+ count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
: Let us make the first part of c.test pass.
echo OK > c.pass
test ! -r b.run
test -f c.run
test -f d.run
- do_count pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
+ count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
: Let us make also the second part of c.test pass.
echo KO > c.xfail
test ! -r b.run
test -f c.run
test -f d.run
- do_count pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
+ count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
: Nothing changed, so only d.test should be run.
for i in 1 2; do
test ! -r b.run
test ! -r c.run
test -f d.run
- do_count pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
+ count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
done
: Let us make d.test run more testcases, and experience _more_ failures.
test ! -r b.run
test ! -r c.run
test -f d.run
- do_count pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
+ count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
: Let us finally make d.test pass.
echo : > d.extra
test ! -r b.run
test ! -r c.run
test -f d.run
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
: All tests have been successful or skipped, nothing should be re-run.
do_recheck --pass
test ! -r b.run
test ! -r c.run
test ! -r d.run
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
cd $srcdir
$AUTOCONF
$AUTOMAKE
-do_count ()
-{
- pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
- eval "$@"
- $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP|ERROR)' stdout || : # For debugging.
- test `grep -c '^PASS:' stdout` -eq $pass
- test `grep -c '^FAIL:' stdout` -eq $fail
- test `grep -c '^XPASS:' stdout` -eq $xpass
- test `grep -c '^XFAIL:' stdout` -eq $xfail
- test `grep -c '^SKIP:' stdout` -eq $skip
- test `grep -c '^ERROR:' stdout` -eq $error
-}
-
for vpath in : false; do
if $vpath; then
mkdir build
test -f a.run
test -f b.run
test -f c.run
- do_count pass=2 fail=1 xpass=1 xfail=0 skip=1 error=0
+ count_test_results total=5 pass=2 fail=1 xpass=1 xfail=0 skip=1 error=0
rm -f *.run
for var in TESTS TEST_LOGS; do
env "$var=" $MAKE -e recheck >stdout || { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test ! -r c.run
env TESTS=a.test $MAKE -e recheck >stdout \
|| { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test ! -r c.run
test ! -r a.run
test -f b.run
test ! -r c.run
- do_count pass=0 fail=0 xpass=0 xfail=1 skip=1 error=0
+ count_test_results total=2 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=0
rm -f *.run
TEST_LOGS=b.log $MAKE -e recheck >stdout \
|| { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test ! -r c.run
TESTS='a.test b.test' $MAKE -e recheck >stdout \
|| { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test ! -r c.run
env TEST_LOGS='a.log c.log' $MAKE -e recheck >stdout \
&& { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
test ! -r a.run
test ! -r b.run
test -f c.run
env TESTS='c.test a.test' $MAKE -e recheck >stdout \
|| { cat stdout; Exit 1; }
cat stdout
- do_count pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+ count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test -f c.run
: succesful.
$MAKE recheck >stdout || { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test ! -r c.run
pass-xpass-fail-xfail-skip-error.t
END
-expected_tests=23
-expected_pass=10
-expected_fail=5
-expected_skip=4
-expected_xfail=2
-expected_xpass=1
-expected_error=1
-
cat > pass.t << 'END'
echo %% pass %%
echo PASS: pass
cat pass4-skip.log
cat pass3-skip2-xfail.log
cat pass-xpass-fail-xfail-skip-error.log
- # For debugging.
- $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP|ERROR)' stdout
- test `grep -c '^PASS:' stdout` -eq $expected_pass
- test `grep -c '^FAIL:' stdout` -eq $expected_fail
- test `grep -c '^XPASS:' stdout` -eq $expected_xpass
- test `grep -c '^XFAIL:' stdout` -eq $expected_xfail
- test `grep -c '^SKIP:' stdout` -eq $expected_skip
- test `grep -c '^ERROR:' stdout` -eq $expected_error
+ count_test_results total=23 pass=10 fail=5 skip=4 xfail=2 xpass=1 error=1
tst=pass-xpass-fail-xfail-skip-error
grep "^PASS: $tst\.t, testcase 1" stdout
grep "^SKIP: $tst\.t, testcase 5" stdout
grep "^ERROR: $tst\.t, testcase 6" stdout
- # Check counts of testcases in testsuite summary printed on console.
- grep "^# TOTAL: *$expected_tests$" stdout
- grep "^# PASS: *$expected_pass$" stdout
- grep "^# XPASS: *$expected_xpass$" stdout
- grep "^# FAIL: *$expected_fail$" stdout
- grep "^# XFAIL: *$expected_xfail$" stdout
- grep "^# SKIP: *$expected_skip$" stdout
- grep "^# ERROR: *$expected_error$" stdout
-
# Check that the content of, and only of, the test logs with at least
# one failing test case has been copied into `test-suite.log'. Note
# that test logs containing skipped or failed test cases are *not*
|| { cat stdout; cat test-suite.log; Exit 1; }
cat test-suite.log
cat stdout
- # For debugging.
- $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP|ERROR)' stdout
- test `grep -c '^PASS:' stdout` -eq 4
- test `grep -c '^SKIP:' stdout` -eq 2
- test `grep -c '^XFAIL:' stdout` -eq 1
- $EGREP '^(FAIL|XPASS|ERROR)' stdout && Exit 1
- # Check counts of testcases in testsuite summary printed on console.
- grep "^# TOTAL: *7$" stdout
- grep "^# PASS: *4$" stdout
- grep "^# XPASS: *0$" stdout
- grep "^# FAIL: *0$" stdout
- grep "^# XFAIL: *1$" stdout
- grep "^# SKIP: *2$" stdout
- grep "^# ERROR: *0$" stdout
+ count_test_results total=7 pass=4 fail=0 skip=2 xfail=1 xpass=0 error=0
cd $srcdir
projects / thirdparty / automake.git / commitdiff
