+2011-08-01 Stefano Lattarini <stefano.lattarini@gmail.com>
+
+ testsuite: better granularity in a couple of tests on TAP support
+ * tests/tap-summary-aux.sh: New auxiliary script, filled with code
+ moved out from ...
+ * tests/tap-summary.test: ... this test, from which the checks on
+ colored testsuite have further been moved into ...
+ * tests/tap-summary-color.test: ... this new test, which in turn
+ * tests/Makefile.am (tap_other_tests): Add `tap-summary-color.test'.
+ (EXTRA_DIST): Distribute `tap-summary-aux.sh'
+ (tap-summary.log): Depend on `tap-summary-aux.sh'.
+ (tap-summary-color.log): Likewise.
+
2011-08-01 Stefano Lattarini <stefano.lattarini@gmail.com>
testsuite: optimize tests on TAP for speed and against duplication
tap-more.test \
tap-more2.test \
tap-recheck.test \
-tap-summary.test
+tap-summary.test \
+tap-summary-color.test
+
+tap-summary.log tap-summary-color.log: tap-summary-aux.sh
+EXTRA_DIST += tap-summary-aux.sh
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
MAINTAINERCLEANFILES = $(parallel_tests) $(instspc_tests)
EXTRA_DIST = ChangeLog-old gen-parallel-tests instspc-tests.sh \
$(TESTS) trivial-test-driver testsuite-summary-checks.sh \
- extract-testsuite-summary tap-setup.sh
+ extract-testsuite-summary tap-setup.sh tap-summary-aux.sh
XFAIL_TESTS = all.test auxdir2.test cond17.test gcj6.test \
override-conditional-2.test pr8365-remake-timing.test \
yacc-dist-nobuild-subdir.test tap-bad-prog.test \
tap-more.test \
tap-more2.test \
tap-recheck.test \
-tap-summary.test
+tap-summary.test \
+tap-summary-color.test
all: all-am
# Their dependencies.
$(tap_with_common_setup_logs): tap-common-setup.log tap-setup.sh
+tap-summary.log tap-summary-color.log: tap-summary-aux.sh
+
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Auxiliary script for tests on TAP support: checking testsuite summary.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+br='============================================================================'
+
+case $use_colors in
+ yes|no) ;;
+ *) fatal_ "invalid \$use_colors value '$use_colors'"
+esac
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat > configure.in <<END
+AC_INIT([GNU AutoTAP], [5.12], [bug-automake@gnu.org])
+AM_INIT_AUTOMAKE([parallel-tests])
+AC_SUBST([PERL], ['$PERL'])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TEST_LOG_COMPILER = cat
+TESTS = all.test
+END
+
+# The following shell variables are influential for this function:
+# - expect_failure
+# - use_colors
+do_check ()
+{
+ case $#,$1 in
+ 1,--pass) expect_failure=no;;
+ 1,--fail) expect_failure=yes;;
+ *) fatal_ "invalid usage of 'do_check'";;
+ esac
+ shift
+ cat > summary.exp
+ cat all.test
+ st=0
+ if test $use_colors = yes; then
+ make_cmd="env AM_COLOR_TESTS=always $MAKE -e"
+ else
+ make_cmd=$MAKE
+ fi
+ $make_cmd check > stdout || st=$?
+ cat stdout
+ if test $expect_failure = yes; then
+ test $st -gt 0 || Exit 1
+ else
+ test $st -eq 0 || Exit 1
+ fi
+ $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \
+ || fatal_ "cannot extract testsuite summary"
+ cat summary.exp
+ cat summary.got
+ if test $use_colors = yes; then
+ # Use cmp, not diff, because the files might contain binary data.
+ compare=cmp
+ else
+ compare=diff
+ fi
+ $compare summary.exp summary.got || Exit 1
+}
+
+if test $use_colors = yes; then
+ red='\e[0;31m'
+ grn='\e[0;32m'
+ lgn='\e[1;32m'
+ blu='\e[1;34m'
+ mgn='\e[0;35m'
+ brg='\e[1m'
+ std='\e[m'
+ echo AUTOMAKE_OPTIONS = color-tests >> Makefile.am
+else
+ red= grn= lgn= blu= mgn= brg= std=
+fi
+
+ success_header="\
+${grn}${br}${std}
+${grn}Testsuite summary for GNU AutoTAP 5.12${std}
+${grn}${br}${std}"
+
+ success_footer=${grn}${br}${std}
+
+ failure_header="\
+${red}${br}${std}
+${red}Testsuite summary for GNU AutoTAP 5.12${std}
+${red}${br}${std}"
+
+ failure_footer="\
+${red}${br}${std}
+${red}See ./test-suite.log${std}
+${red}Please report to bug-automake@gnu.org${std}
+${red}${br}${std}"
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+./configure
+
+# 1 pass.
+{ echo 1..1 && echo ok; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 skip.
+{ echo 1..1 && echo 'ok # SKIP'; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 xfail.
+{ echo 1..1 && echo 'not ok # TODO'; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 fail.
+{ echo 1..1 && echo not ok; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 xpass.
+{ echo 1..1 && echo 'ok # TODO'; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+# 1 hard error.
+{ echo 1..1 && echo 'Bail out!'; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# 3 non-failing results.
+cat > all.test <<END
+1..3
+ok
+not ok # TODO
+ok # SKIP
+END
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 pass, 1 skip, 1 fail.
+cat > all.test <<END
+1..3
+ok
+ok # SKIP
+not ok
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 pass, 1 xfail, 1 xpass.
+cat > all.test <<END
+1..3
+ok
+ok # TODO
+not ok # TODO
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+# 1 skip, 1 xfail, 1 error.
+cat > all.test <<END
+1..3
+ok # SKIP
+not ok # TODO
+Bail out!
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# 1 of each kind
+cat > all.test <<END
+1..6
+ok
+not ok
+ok # TODO
+not ok # TODO
+ok # SKIP
+Bail out!
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 6${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+${red}# FAIL: 1${std}
+${red}# XPASS: 1${std}
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# Prepare some common data for later.
+for i in 0 1 2 3 4 5 6 7 8 9; do
+ for j in 0 1 2 3 4 5 6 7 8 9; do
+ echo "ok"
+ echo "not ok # TODO"
+ echo "ok # SKIP"
+ done
+done > tap
+
+# Lots of non-failures (300 per kind).
+(cat tap && cat tap && cat tap) > all.test
+test `wc -l <all.test` -eq 900 || Exit 99 # Sanity check.
+echo 1..900 >> all.test # Test plan.
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 900${std}
+${grn}# PASS: 300${std}
+${blu}# SKIP: 300${std}
+${lgn}# XFAIL: 300${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 failure and lots of non-failures means failure.
+(cat tap && echo "not ok" && cat tap) > all.test
+test `wc -l <all.test` -eq 601 || Exit 99 # Sanity check.
+echo 1..601 >> all.test # Test plan.
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 601${std}
+${grn}# PASS: 200${std}
+${blu}# SKIP: 200${std}
+${lgn}# XFAIL: 200${std}
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 error and lots of non-failures means failure.
+(cat tap && sed 30q tap && echo 'Bail out!') > all.test
+test `wc -l <all.test` -eq 331 || Exit 99 # Sanity check.
+echo 1..331 >> all.test # Test plan.
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 331${std}
+${grn}# PASS: 110${std}
+${blu}# SKIP: 110${std}
+${lgn}# XFAIL: 110${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+:
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - colorized testsuite summary
+
+. ./defs-static
+
+use_colors=yes
+. "$testsrcdir"/tap-summary-aux.sh
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# TAP support:
-# - testsuite summary
# - colorized testsuite summary
-# TODO: splitting this into two distinct test scripts might be worthwhile
-parallel_tests=yes
-. ./defs || Exit 1
+. ./defs-static
-cp "$top_testsrcdir"/lib/tap-driver . \
- || fatal_ "failed to fetch auxiliary script tap-driver"
-
-cat > configure.in <<END
-AC_INIT([GNU AutoTAP], [5.12], [bug-automake@gnu.org])
-AM_INIT_AUTOMAKE([parallel-tests])
-AC_SUBST([PERL], ['$PERL'])
-AC_CONFIG_FILES([Makefile])
-AC_OUTPUT
-END
-
-cat > Makefile.stub << 'END'
-TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
-TEST_LOG_COMPILER = cat
-TESTS = all.test
-END
-
-$ACLOCAL
-$AUTOCONF
-
-br='============================================================================'
-
-# The following shell variables are influential for this function:
-# - expect_failure
-# - use_colors
-do_check ()
-{
- case $#,$1 in
- 1,--pass) expect_failure=no;;
- 1,--fail) expect_failure=yes;;
- *) fatal_ "invalid usage of 'do_check'";;
- esac
- shift
- cat > summary.exp
- cat all.test
- st=0
- if test $use_colors = yes; then
- make_cmd="env AM_COLOR_TESTS=always $MAKE -e"
- else
- make_cmd=$MAKE
- fi
- $make_cmd check > stdout || st=$?
- cat stdout
- if test $expect_failure = yes; then
- test $st -gt 0 || Exit 1
- else
- test $st -eq 0 || Exit 1
- fi
- $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \
- || fatal_ "cannot extract testsuite summary"
- cat summary.exp
- cat summary.got
- if test $use_colors = yes; then
- # Use cmp, not diff, because the files might contain binary data.
- compare=cmp
- else
- compare=diff
- fi
- $compare summary.exp summary.got || Exit 1
-}
-
-for use_colors in "no" "yes"; do
-
- cp Makefile.stub Makefile.am
-
- if test $use_colors = yes; then
- red='\e[0;31m'
- grn='\e[0;32m'
- lgn='\e[1;32m'
- blu='\e[1;34m'
- mgn='\e[0;35m'
- brg='\e[1m'
- std='\e[m'
- echo AUTOMAKE_OPTIONS = color-tests >> Makefile.am
- else
- red= grn= lgn= blu= mgn= brg= std=
- fi
-
- success_header="\
-${grn}${br}${std}
-${grn}Testsuite summary for GNU AutoTAP 5.12${std}
-${grn}${br}${std}"
-
- success_footer=${grn}${br}${std}
-
- failure_header="\
-${red}${br}${std}
-${red}Testsuite summary for GNU AutoTAP 5.12${std}
-${red}${br}${std}"
-
- failure_footer="\
-${red}${br}${std}
-${red}See ./test-suite.log${std}
-${red}Please report to bug-automake@gnu.org${std}
-${red}${br}${std}"
-
- $AUTOMAKE --add-missing
-
- ./configure
-
- # 1 pass.
- { echo 1..1 && echo ok; } > all.test
- do_check --pass <<END
-$success_header
-${brg}# TOTAL: 1${std}
-${grn}# PASS: 1${std}
-# SKIP: 0
-# XFAIL: 0
-# FAIL: 0
-# XPASS: 0
-# ERROR: 0
-$success_footer
-END
-
- # 1 skip.
- { echo 1..1 && echo 'ok # SKIP'; } > all.test
- do_check --pass <<END
-$success_header
-${brg}# TOTAL: 1${std}
-# PASS: 0
-${blu}# SKIP: 1${std}
-# XFAIL: 0
-# FAIL: 0
-# XPASS: 0
-# ERROR: 0
-$success_footer
-END
-
- # 1 xfail.
- { echo 1..1 && echo 'not ok # TODO'; } > all.test
- do_check --pass <<END
-$success_header
-${brg}# TOTAL: 1${std}
-# PASS: 0
-# SKIP: 0
-${lgn}# XFAIL: 1${std}
-# FAIL: 0
-# XPASS: 0
-# ERROR: 0
-$success_footer
-END
-
- # 1 fail.
- { echo 1..1 && echo not ok; } > all.test
- do_check --fail <<END
-$failure_header
-${brg}# TOTAL: 1${std}
-# PASS: 0
-# SKIP: 0
-# XFAIL: 0
-${red}# FAIL: 1${std}
-# XPASS: 0
-# ERROR: 0
-$failure_footer
-END
-
- # 1 xpass.
- { echo 1..1 && echo 'ok # TODO'; } > all.test
- do_check --fail <<END
-$failure_header
-${brg}# TOTAL: 1${std}
-# PASS: 0
-# SKIP: 0
-# XFAIL: 0
-# FAIL: 0
-${red}# XPASS: 1${std}
-# ERROR: 0
-$failure_footer
-END
-
- # 1 hard error.
- { echo 1..1 && echo 'Bail out!'; } > all.test
- do_check --fail <<END
-$failure_header
-${brg}# TOTAL: 1${std}
-# PASS: 0
-# SKIP: 0
-# XFAIL: 0
-# FAIL: 0
-# XPASS: 0
-${mgn}# ERROR: 1${std}
-$failure_footer
-END
-
- # 3 non-failing results.
- cat > all.test <<END
-1..3
-ok
-not ok # TODO
-ok # SKIP
-END
- do_check --pass <<END
-$success_header
-${brg}# TOTAL: 3${std}
-${grn}# PASS: 1${std}
-${blu}# SKIP: 1${std}
-${lgn}# XFAIL: 1${std}
-# FAIL: 0
-# XPASS: 0
-# ERROR: 0
-$success_footer
-END
-
- # 1 pass, 1 skip, 1 fail.
- cat > all.test <<END
-1..3
-ok
-ok # SKIP
-not ok
-END
- do_check --fail <<END
-$failure_header
-${brg}# TOTAL: 3${std}
-${grn}# PASS: 1${std}
-${blu}# SKIP: 1${std}
-# XFAIL: 0
-${red}# FAIL: 1${std}
-# XPASS: 0
-# ERROR: 0
-$failure_footer
-END
-
- # 1 pass, 1 xfail, 1 xpass.
- cat > all.test <<END
-1..3
-ok
-ok # TODO
-not ok # TODO
-END
- do_check --fail <<END
-$failure_header
-${brg}# TOTAL: 3${std}
-${grn}# PASS: 1${std}
-# SKIP: 0
-${lgn}# XFAIL: 1${std}
-# FAIL: 0
-${red}# XPASS: 1${std}
-# ERROR: 0
-$failure_footer
-END
-
- # 1 skip, 1 xfail, 1 error.
- cat > all.test <<END
-1..3
-ok # SKIP
-not ok # TODO
-Bail out!
-END
- do_check --fail <<END
-$failure_header
-${brg}# TOTAL: 3${std}
-# PASS: 0
-${blu}# SKIP: 1${std}
-${lgn}# XFAIL: 1${std}
-# FAIL: 0
-# XPASS: 0
-${mgn}# ERROR: 1${std}
-$failure_footer
-END
-
- # 1 of each kind
- cat > all.test <<END
-1..6
-ok
-not ok
-ok # TODO
-not ok # TODO
-ok # SKIP
-Bail out!
-END
- do_check --fail <<END
-$failure_header
-${brg}# TOTAL: 6${std}
-${grn}# PASS: 1${std}
-${blu}# SKIP: 1${std}
-${lgn}# XFAIL: 1${std}
-${red}# FAIL: 1${std}
-${red}# XPASS: 1${std}
-${mgn}# ERROR: 1${std}
-$failure_footer
-END
-
- # Prepare some common data for later.
- for i in 0 1 2 3 4 5 6 7 8 9; do
- for j in 0 1 2 3 4 5 6 7 8 9; do
- echo "ok"
- echo "not ok # TODO"
- echo "ok # SKIP"
- done
- done > tap
-
- # Lots of non-failures (300 per kind).
- (cat tap && cat tap && cat tap) > all.test
- test `wc -l <all.test` -eq 900 || Exit 99 # Sanity check.
- echo 1..900 >> all.test # Test plan.
- do_check --pass <<END
-$success_header
-${brg}# TOTAL: 900${std}
-${grn}# PASS: 300${std}
-${blu}# SKIP: 300${std}
-${lgn}# XFAIL: 300${std}
-# FAIL: 0
-# XPASS: 0
-# ERROR: 0
-$success_footer
-END
-
- # 1 failure and lots of non-failures means failure.
- (cat tap && echo "not ok" && cat tap) > all.test
- test `wc -l <all.test` -eq 601 || Exit 99 # Sanity check.
- echo 1..601 >> all.test # Test plan.
- do_check --fail <<END
-$failure_header
-${brg}# TOTAL: 601${std}
-${grn}# PASS: 200${std}
-${blu}# SKIP: 200${std}
-${lgn}# XFAIL: 200${std}
-${red}# FAIL: 1${std}
-# XPASS: 0
-# ERROR: 0
-$failure_footer
-END
-
- # 1 error and lots of non-failures means failure.
- (cat tap && sed 30q tap && echo 'Bail out!') > all.test
- test `wc -l <all.test` -eq 331 || Exit 99 # Sanity check.
- echo 1..331 >> all.test # Test plan.
- do_check --fail <<END
-$failure_header
-${brg}# TOTAL: 331${std}
-${grn}# PASS: 110${std}
-${blu}# SKIP: 110${std}
-${lgn}# XFAIL: 110${std}
-# FAIL: 0
-# XPASS: 0
-${mgn}# ERROR: 1${std}
-$failure_footer
-END
-
-done # use_colors in "yes" "no"
-
-:
+use_colors=no
+. "$testsrcdir"/tap-summary-aux.sh
projects / thirdparty / automake.git / commitdiff
