+2011-07-07 Stefano Lattarini <stefano.lattarini@gmail.com>
+
+ parallel-tests: simplify testsuite summary
+ Prefer a more deterministic, "tabular" format for the testsuite
+ summary, always listing the numbers of passed, failed, xfailed,
+ xpassed, skipped and errored tests, even when these numbers are
+ zero. This simplify the logic of testsuite summary creation,
+ makes it more easily machine-parseable, and will probably allow
+ for easier addition of new kinds of test results in the future.
+ * lib/am/check.am (am__tty_colors_dummy): New make variable, to
+ reduce code duplication. Extracted from previous versions of
+ $(am__tty_colors), and extended by defining two new variables
+ `$mgn' and `$brg'.
+ [%?COLOR%, %!?COLOR%] (am__tty_colors): Use that new variable.
+ (am__text_box): Delete, is not needed anymore.
+ ($(TEST_SUITE_LOG)): Rewrite associated rules to implement the
+ new testsuite summary format.
+ * NEWS: Update.
+ * tests/check10.test: Don't run with the parallel-tests harness
+ too, that makes no sense anymore.
+ * tests/color.test: Update and adjust.
+ * tests/color2.test: Likewise.
+ * tests/parallel-tests.test: Likewise.
+ * tests/parallel-tests3.test: Likewise.
+ * tests/parallel-tests6.test: Likewise.
+ * tests/parallel-tests9.test: Likewise.
+ * tests/parallel-tests-unreadable-log.test: Likewise.
+ * tests/parallel-tests-empty-testlogs.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-no-spurious-summary.test: Likewise.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-end-test-results.test: Likewise.
+ * tests/parallel-tests-no-color-in-log.test: New test.
+ * tests/testsuite-summary-color.test: Likewise.
+ * tests/testsuite-summary-count.test: Likewise.
+ * tests/testsuite-summary-count-many.test: Likewise.
+ * tests/testsuite-summary-reference-log.test: Likewise.
+ * tests/testsuite-summary-checks.sh: New auxiliary script, used
+ by the new tests above.
+ * tests/extract-testsuite-summary: Likewise.
+ * tests/trivial-test-driver: Optimize for speed when there are
+ lots of of tests.
+ * tests/Makefile.am (EXTRA_DIST): Distribute them.
+ (testsuite-summary-color.log, testsuite-summary-count.log): Depend
+ on them.
+ (testsuite-summary-count-many.log): Depend on the auxiliary scripts
+ 'trivial-test-driver' and 'extract-testsuite-summary'.
+ (TESTS): Update.
+
2011-07-07 Stefano Lattarini <stefano.lattarini@gmail.com>
parallel-tests: new recognized test result 'ERROR'
as failures even when the test originating them was listed in
XFAIL_TESTS).
+ - The testsuite summary displayed by the parallel-test harness has a
+ completely new format, that always listing the numbers of passed,
+ failed, xfailed, xpassed, skipped and errored tests, even when these
+ numbers are zero (but using smart coloring when the color-tests option
+ is in effect).
+
- The default testsuite driver offered by the 'parallel-tests' option is
now implemented (partly at least) with the help of automake-provided
auxiliary scripts (e.g., `test-driver'), instead of relying entirely
am__v_at_0 = @
SOURCES =
DIST_SOURCES =
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
-am__color_tests=no; \
-red=; grn=; lgn=; blu=; std=; \
+$(am__tty_colors_dummy); \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
grn='\e[0;32m'; \
lgn='\e[1;32m'; \
blu='\e[1;34m'; \
+ mgn='\e[0;35m'; \
+ brg='\e[1m'; \
std='\e[m'; \
}
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
# Restructured Text title.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
-# Put stdin (possibly several lines separated by ". ") in a box.
-# Prefix each line by 'col' and terminate each with 'std', for coloring.
-# Multi line coloring is problematic with "less -R", so we really need
-# to color each line individually.
-am__text_box = $(AWK) '{ \
- n = split($$0, lines, "\\. "); max = 0; \
- for (i = 1; i <= n; ++i) \
- if (max < length(lines[i])) \
- max = length(lines[i]); \
- for (i = 0; i < max; ++i) \
- line = line "="; \
- print col line std; \
- for (i = 1; i <= n; ++i) \
- if (lines[i]) \
- print col lines[i] std; \
- print col line std; \
-}'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
+ @$(am__sh_e_setup); $(am__tty_colors); \
rst_magic=":test-result:"; \
nlinit=`echo 'nl="'; echo '"'`; eval "$$nlinit"; unset nlinit; \
list='$(TEST_LOGS)'; \
exec 0<&5; \
done; \
exec 5<&-; \
- results=`echo "$$results1" && echo "$$results2"` \
+ results=`echo "$$results1" && echo "$$results2"`; \
all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \
fail=`echo "$$results" | grep -c '^FAIL'`; \
pass=`echo "$$results" | grep -c '^PASS'`; \
xfail=`echo "$$results" | grep -c '^XFAIL'`; \
xpass=`echo "$$results" | grep -c '^XPASS'`; \
error=`echo "$$results" | grep -c '^ERROR'`; \
- fail=`expr $$fail + $$error`; \
- failures=`expr $$fail + $$xpass`; \
- all=`expr $$all - $$skip`; \
- if test "$$all" -eq 1; then tests=test; All=; \
- else tests=tests; All="All "; fi; \
- case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
- fail=0:xpass=0:xfail=0) \
- msg="$$All$$all $$tests passed. "; \
- exit=true;; \
- fail=0:xpass=0:xfail=*) \
- msg="$$All$$all $$tests behaved as expected"; \
- if test "$$xfail" -eq 1; then xfailures=failure; \
- else xfailures=failures; fi; \
- msg="$$msg ($$xfail expected $$xfailures). "; \
- exit=true;; \
- fail=*:xpass=0:xfail=*) \
- msg="$$fail of $$all $$tests failed. "; \
- exit=false;; \
- fail=*:xpass=*:xfail=*) \
- msg="$$failures of $$all $$tests did not behave as expected"; \
- if test "$$xpass" -eq 1; then xpasses=pass; \
- else xpasses=passes; fi; \
- msg="$$msg ($$xpass unexpected $$xpasses). "; \
- exit=false;; \
- *) \
- echo >&2 "incorrect case"; exit 4;; \
- esac; \
- if test "$$skip" -ne 0; then \
- if test "$$skip" -eq 1; then \
- msg="$$msg($$skip test was not run). "; \
- else \
- msg="$$msg($$skip tests were not run). "; \
- fi; \
- fi; \
+ if test `expr $$fail + $$xpass + $$error` -eq 0; then \
+ success=true; \
+ else \
+ success=false; \
+ fi; \
+ br='==================='; br=$$br$$br$$br$$br; \
+ result_count () \
+ { \
+ if test x"$$1" = x"--maybe-color"; then \
+ maybe_colorize=yes; \
+ elif test x"$$1" = x"--no-color"; then \
+ maybe_colorize=no; \
+ else \
+ echo "$@: invalid 'result_count' usage" >&2; exit 4; \
+ fi; \
+ shift; \
+ desc=$$1 count=$$2; \
+ if test $$maybe_colorize = yes && test $$count -gt 0; then \
+ color_start=$$3 color_end=$$std; \
+ else \
+ color_start= color_end=; \
+ fi; \
+ echo "$${color_start}# $$desc $$count$${color_end}"; \
+ }; \
+ create_testsuite_report () \
+ { \
+ result_count $$1 "TOTAL:" $$all "$$brg"; \
+ result_count $$1 "PASS: " $$pass "$$grn"; \
+ result_count $$1 "SKIP: " $$skip "$$blu"; \
+ result_count $$1 "XFAIL:" $$xfail "$$lgn"; \
+ result_count $$1 "FAIL: " $$fail "$$red"; \
+ result_count $$1 "XPASS:" $$xpass "$$red"; \
+ result_count $$1 "ERROR:" $$error "$$mgn"; \
+ }; \
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
- echo "$$msg"; \
+ create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
- if test "$$failures" -ne 0; then \
- msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
- if test -n "$(PACKAGE_BUGREPORT)"; then \
- msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
- fi; \
- fi; \
- test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
- $(am__tty_colors); \
- if $$exit; then \
+ if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
+ test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
+ fi; \
+ echo "$${col}$$br$${std}"; \
+ echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}$$br$${std}"; \
+ create_testsuite_report --maybe-color; \
+ echo "$$col$$br$$std"; \
+ if $$success; then :; else \
+ echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
+ fi; \
+ echo "$$col$$br$$std"; \
fi; \
- echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \
- $$exit
+ $$success
# Run all the tests.
check-TESTS:
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
if %?COLOR%
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
-am__color_tests=no; \
-red=; grn=; lgn=; blu=; std=; \
+$(am__tty_colors_dummy); \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
grn='\e[0;32m'; \
lgn='\e[1;32m'; \
blu='\e[1;34m'; \
+ mgn='\e[0;35m'; \
+ brg='\e[1m'; \
std='\e[m'; \
}
else !%?COLOR%
-am__tty_colors = red= grn= lgn= blu= std= am__color_tests=no
+am__tty_colors = $(am__tty_colors_dummy)
endif !%?COLOR%
.PHONY: check-TESTS
# Restructured Text title.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
-# Put stdin (possibly several lines separated by ". ") in a box.
-# Prefix each line by 'col' and terminate each with 'std', for coloring.
-# Multi line coloring is problematic with "less -R", so we really need
-# to color each line individually.
-am__text_box = $(AWK) '{ \
- n = split($$0, lines, "\\. "); max = 0; \
- for (i = 1; i <= n; ++i) \
- if (max < length(lines[i])) \
- max = length(lines[i]); \
- for (i = 0; i < max; ++i) \
- line = line "="; \
- print col line std; \
- for (i = 1; i <= n; ++i) \
- if (lines[i]) \
- print col lines[i] std; \
- print col line std; \
-}'
-
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT)
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
+ @$(am__sh_e_setup); $(am__tty_colors); \
## The custom reStructuredText field used to register the outcome of
## a test. This is for supporting test protocols, such as TAP and
## SubUnit, where a single test script can run multiple tests, each
done; \
## We don't need this anymore.
exec 5<&-; \
-## Prepare the test suite summary.
- results=`echo "$$results1" && echo "$$results2"` \
+## Prepare data for the test suite summary.
+ results=`echo "$$results1" && echo "$$results2"`; \
all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \
fail=`echo "$$results" | grep -c '^FAIL'`; \
pass=`echo "$$results" | grep -c '^PASS'`; \
xfail=`echo "$$results" | grep -c '^XFAIL'`; \
xpass=`echo "$$results" | grep -c '^XPASS'`; \
error=`echo "$$results" | grep -c '^ERROR'`; \
-## FIXME: for the moment, we count errors as failures, otherwise the code
-## that displays the testsuite summary will become too complicated.
- fail=`expr $$fail + $$error`; \
- failures=`expr $$fail + $$xpass`; \
- all=`expr $$all - $$skip`; \
- if test "$$all" -eq 1; then tests=test; All=; \
- else tests=tests; All="All "; fi; \
- case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
- fail=0:xpass=0:xfail=0) \
- msg="$$All$$all $$tests passed. "; \
- exit=true;; \
- fail=0:xpass=0:xfail=*) \
- msg="$$All$$all $$tests behaved as expected"; \
- if test "$$xfail" -eq 1; then xfailures=failure; \
- else xfailures=failures; fi; \
- msg="$$msg ($$xfail expected $$xfailures). "; \
- exit=true;; \
- fail=*:xpass=0:xfail=*) \
- msg="$$fail of $$all $$tests failed. "; \
- exit=false;; \
- fail=*:xpass=*:xfail=*) \
- msg="$$failures of $$all $$tests did not behave as expected"; \
- if test "$$xpass" -eq 1; then xpasses=pass; \
- else xpasses=passes; fi; \
- msg="$$msg ($$xpass unexpected $$xpasses). "; \
- exit=false;; \
- *) \
- echo >&2 "incorrect case"; exit 4;; \
- esac; \
- if test "$$skip" -ne 0; then \
- if test "$$skip" -eq 1; then \
- msg="$$msg($$skip test was not run). "; \
- else \
- msg="$$msg($$skip tests were not run). "; \
- fi; \
- fi; \
+## Whether the testsuite was successful or not.
+ if test `expr $$fail + $$xpass + $$error` -eq 0; then \
+ success=true; \
+ else \
+ success=false; \
+ fi; \
+## Make $br a line of exactly 76 `=' characters, that will be used to
+## enclose the testsuite summary report when displayed on the console.
+ br='==================='; br=$$br$$br$$br$$br; \
+## When writing the test summary to the console, we want to color a line
+## reporting the count of some result *only* if at least one test
+## experienced such a result. This function is handy in this regard.
+ result_count () \
+ { \
+ if test x"$$1" = x"--maybe-color"; then \
+ maybe_colorize=yes; \
+ elif test x"$$1" = x"--no-color"; then \
+ maybe_colorize=no; \
+ else \
+ echo "$@: invalid 'result_count' usage" >&2; exit 4; \
+ fi; \
+ shift; \
+ desc=$$1 count=$$2; \
+ if test $$maybe_colorize = yes && test $$count -gt 0; then \
+ color_start=$$3 color_end=$$std; \
+ else \
+ color_start= color_end=; \
+ fi; \
+ echo "$${color_start}# $$desc $$count$${color_end}"; \
+ }; \
+## A shell function that creates the testsuite summary. We need it
+## because we have to create *two* summaries, one for test-suite.log,
+## and a possibly-colorized one for console output.
+ create_testsuite_report () \
+ { \
+ result_count $$1 "TOTAL:" $$all "$$brg"; \
+ result_count $$1 "PASS: " $$pass "$$grn"; \
+ result_count $$1 "SKIP: " $$skip "$$blu"; \
+ result_count $$1 "XFAIL:" $$xfail "$$lgn"; \
+ result_count $$1 "FAIL: " $$fail "$$red"; \
+ result_count $$1 "XPASS:" $$xpass "$$red"; \
+ result_count $$1 "ERROR:" $$error "$$mgn"; \
+ }; \
## Write "global" testsuite log.
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
- echo "$$msg"; \
+ create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
- if test "$$failures" -ne 0; then \
- msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
- if test -n "$(PACKAGE_BUGREPORT)"; then \
- msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
- fi; \
- fi; \
- test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
-## Emit the test summary on the console, and exit.
- $(am__tty_colors); \
- if $$exit; then \
+## Emit the test summary on the console.
+ if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
+ test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
+ fi; \
+## Multi line coloring is problematic with "less -R", so we really need
+## to color each line individually.
+ echo "$${col}$$br$${std}"; \
+ echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}$$br$${std}"; \
+## This is expected to go to the console, so it might have to be colorized.
+ create_testsuite_report --maybe-color; \
+ echo "$$col$$br$$std"; \
+ if $$success; then :; else \
+ echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
+ fi; \
+ echo "$$col$$br$$std"; \
fi; \
- echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \
- $$exit
+## Be sure to exit with the proper exit status.
+ $$success
RECHECK_LOGS = $(TEST_LOGS)
parallel-tests-no-spurious-summary.test \
parallel-tests-exit-statuses.test \
parallel-tests-console-output.test \
+testsuite-summary-color.test \
+testsuite-summary-count.test \
+testsuite-summary-count-many.test \
+testsuite-summary-reference-log.test \
test-driver-end-test-results.test \
test-driver-custom-no-extra-driver.test \
test-driver-custom.test \
test-driver-custom-html.log: trivial-test-driver
EXTRA_DIST += trivial-test-driver
+testsuite-summary-color.log testsuite-summary-count.log: \
+ testsuite-summary-checks.sh extract-testsuite-summary
+EXTRA_DIST += testsuite-summary-checks.sh
+EXTRA_DIST += extract-testsuite-summary
+
+testsuite-summary-count-many.log: trivial-test-driver
+testsuite-summary-count-many.log: extract-testsuite-summary
+
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
am__v_at_0 = @
SOURCES =
DIST_SOURCES =
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
-am__color_tests=no; \
-red=; grn=; lgn=; blu=; std=; \
+$(am__tty_colors_dummy); \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
grn='\e[0;32m'; \
lgn='\e[1;32m'; \
blu='\e[1;34m'; \
+ mgn='\e[0;35m'; \
+ brg='\e[1m'; \
std='\e[m'; \
}
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
# Restructured Text title.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
-# Put stdin (possibly several lines separated by ". ") in a box.
-# Prefix each line by 'col' and terminate each with 'std', for coloring.
-# Multi line coloring is problematic with "less -R", so we really need
-# to color each line individually.
-am__text_box = $(AWK) '{ \
- n = split($$0, lines, "\\. "); max = 0; \
- for (i = 1; i <= n; ++i) \
- if (max < length(lines[i])) \
- max = length(lines[i]); \
- for (i = 0; i < max; ++i) \
- line = line "="; \
- print col line std; \
- for (i = 1; i <= n; ++i) \
- if (lines[i]) \
- print col lines[i] std; \
- print col line std; \
-}'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
top_srcdir = @top_srcdir@
MAINTAINERCLEANFILES = $(parallel_tests) $(instspc_tests)
EXTRA_DIST = ChangeLog-old gen-parallel-tests instspc-tests.sh \
- $(TESTS) trivial-test-driver
+ $(TESTS) trivial-test-driver testsuite-summary-checks.sh \
+ extract-testsuite-summary
XFAIL_TESTS = all.test auxdir2.test cond17.test gcj6.test \
override-conditional-2.test pr8365-remake-timing.test \
yacc-dist-nobuild-subdir.test txinfo5.test \
$(instspc_xfail_tests)
parallel_tests = backcompat5-p.test check-exported-srcdir-p.test \
check-tests-in-builddir-p.test check-tests_environment-p.test \
- check-p.test check10-p.test check11-p.test check12-p.test \
- check2-p.test check3-p.test check4-p.test check5-p.test \
- check6-p.test check7-p.test check8-p.test check9-p.test \
- color-p.test color2-p.test comment9-p.test dejagnu-p.test \
- exeext4-p.test maken3-p.test maken4-p.test \
- posixsubst-tests-p.test
+ check-p.test check11-p.test check12-p.test check2-p.test \
+ check3-p.test check4-p.test check5-p.test check6-p.test \
+ check7-p.test check8-p.test check9-p.test color-p.test \
+ color2-p.test comment9-p.test dejagnu-p.test exeext4-p.test \
+ maken3-p.test maken4-p.test posixsubst-tests-p.test
instspc_tests = instspc-squote-build.test instspc-squote-install.test \
instspc-dquote-build.test instspc-dquote-install.test \
instspc-bquote-build.test instspc-bquote-install.test \
parallel-tests-no-spurious-summary.test \
parallel-tests-exit-statuses.test \
parallel-tests-console-output.test \
+testsuite-summary-color.test \
+testsuite-summary-count.test \
+testsuite-summary-count-many.test \
+testsuite-summary-reference-log.test \
test-driver-end-test-results.test \
test-driver-custom-no-extra-driver.test \
test-driver-custom.test \
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
+ @$(am__sh_e_setup); $(am__tty_colors); \
rst_magic=":test-result:"; \
nlinit=`echo 'nl="'; echo '"'`; eval "$$nlinit"; unset nlinit; \
list='$(TEST_LOGS)'; \
exec 0<&5; \
done; \
exec 5<&-; \
- results=`echo "$$results1" && echo "$$results2"` \
+ results=`echo "$$results1" && echo "$$results2"`; \
all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \
fail=`echo "$$results" | grep -c '^FAIL'`; \
pass=`echo "$$results" | grep -c '^PASS'`; \
xfail=`echo "$$results" | grep -c '^XFAIL'`; \
xpass=`echo "$$results" | grep -c '^XPASS'`; \
error=`echo "$$results" | grep -c '^ERROR'`; \
- fail=`expr $$fail + $$error`; \
- failures=`expr $$fail + $$xpass`; \
- all=`expr $$all - $$skip`; \
- if test "$$all" -eq 1; then tests=test; All=; \
- else tests=tests; All="All "; fi; \
- case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
- fail=0:xpass=0:xfail=0) \
- msg="$$All$$all $$tests passed. "; \
- exit=true;; \
- fail=0:xpass=0:xfail=*) \
- msg="$$All$$all $$tests behaved as expected"; \
- if test "$$xfail" -eq 1; then xfailures=failure; \
- else xfailures=failures; fi; \
- msg="$$msg ($$xfail expected $$xfailures). "; \
- exit=true;; \
- fail=*:xpass=0:xfail=*) \
- msg="$$fail of $$all $$tests failed. "; \
- exit=false;; \
- fail=*:xpass=*:xfail=*) \
- msg="$$failures of $$all $$tests did not behave as expected"; \
- if test "$$xpass" -eq 1; then xpasses=pass; \
- else xpasses=passes; fi; \
- msg="$$msg ($$xpass unexpected $$xpasses). "; \
- exit=false;; \
- *) \
- echo >&2 "incorrect case"; exit 4;; \
- esac; \
- if test "$$skip" -ne 0; then \
- if test "$$skip" -eq 1; then \
- msg="$$msg($$skip test was not run). "; \
- else \
- msg="$$msg($$skip tests were not run). "; \
- fi; \
- fi; \
+ if test `expr $$fail + $$xpass + $$error` -eq 0; then \
+ success=true; \
+ else \
+ success=false; \
+ fi; \
+ br='==================='; br=$$br$$br$$br$$br; \
+ result_count () \
+ { \
+ if test x"$$1" = x"--maybe-color"; then \
+ maybe_colorize=yes; \
+ elif test x"$$1" = x"--no-color"; then \
+ maybe_colorize=no; \
+ else \
+ echo "$@: invalid 'result_count' usage" >&2; exit 4; \
+ fi; \
+ shift; \
+ desc=$$1 count=$$2; \
+ if test $$maybe_colorize = yes && test $$count -gt 0; then \
+ color_start=$$3 color_end=$$std; \
+ else \
+ color_start= color_end=; \
+ fi; \
+ echo "$${color_start}# $$desc $$count$${color_end}"; \
+ }; \
+ create_testsuite_report () \
+ { \
+ result_count $$1 "TOTAL:" $$all "$$brg"; \
+ result_count $$1 "PASS: " $$pass "$$grn"; \
+ result_count $$1 "SKIP: " $$skip "$$blu"; \
+ result_count $$1 "XFAIL:" $$xfail "$$lgn"; \
+ result_count $$1 "FAIL: " $$fail "$$red"; \
+ result_count $$1 "XPASS:" $$xpass "$$red"; \
+ result_count $$1 "ERROR:" $$error "$$mgn"; \
+ }; \
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
- echo "$$msg"; \
+ create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
- if test "$$failures" -ne 0; then \
- msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
- if test -n "$(PACKAGE_BUGREPORT)"; then \
- msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
- fi; \
- fi; \
- test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
- $(am__tty_colors); \
- if $$exit; then \
+ if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
+ test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
+ fi; \
+ echo "$${col}$$br$${std}"; \
+ echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}$$br$${std}"; \
+ create_testsuite_report --maybe-color; \
+ echo "$$col$$br$$std"; \
+ if $$success; then :; else \
+ echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
+ fi; \
+ echo "$$col$$br$$std"; \
fi; \
- echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \
- $$exit
+ $$success
# Run all the tests.
check-TESTS:
check-tests-in-builddir-p.log: check-tests-in-builddir.test
check-tests_environment-p.log: check-tests_environment.test
check-p.log: check.test
-check10-p.log: check10.test
check11-p.log: check11.test
check12-p.log: check12.test
check2-p.log: check2.test
test-driver-custom-multitest-recheck2.log: trivial-test-driver
test-driver-custom-html.log: trivial-test-driver
+testsuite-summary-color.log testsuite-summary-count.log: \
+ testsuite-summary-checks.sh extract-testsuite-summary
+
+testsuite-summary-count-many.log: trivial-test-driver
+testsuite-summary-count-many.log: extract-testsuite-summary
+
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
# Check singular and plural in test summaries.
+# This test only makes sense for the older serial testsuite driver.
+parallel_tests=no
. ./defs || Exit 1
cat >> configure.in << 'END'
# Keep this in sync with lib/am/check.am:$(am__color_tests).
if $FGREP '= Xalways || test -t 1 ' stdout; then
# Extra verbose make, resort to laxer checks.
+ # Note that we also want to check that the testsuite summary is
+ # not unduly colorized.
(
set +e # In case some grepped regex below isn't matched.
# Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "TOTAL.*:"
cat stdout | grep "PASS.*:"
cat stdout | grep "FAIL.*:"
cat stdout | grep "SKIP.*:"
cat stdout | grep "XFAIL.*:"
cat stdout | grep "XPASS.*:"
cat stdout | grep "ERROR.*:"
- # To check that the testsuite summary is not unduly colorized.
- cat stdout | grep '===='
cat stdout | grep 'test.*expected'
cat stdout | grep 'test.*not run'
+ cat stdout | grep '===='
+ cat stdout | grep '[Ss]ee .*test-suite\.log'
+ cat stdout | grep '[Tt]estsuite summary'
) | grep "$esc" && Exit 1
: For shells with broken 'set -e'
else
# Keep this in sync with lib/am/check.am:$(am__color_tests).
if $FGREP '= Xalways || test -t 1 ' stdout; then
# Extra verbose make, resort to laxer checks.
+ # Note that we also want to check that the testsuite summary is
+ # not unduly colorized.
(
set +e # In case some grepped regex below isn't matched.
# Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "TOTAL.*:"
cat stdout | grep "PASS.*:"
cat stdout | grep "FAIL.*:"
cat stdout | grep "SKIP.*:"
cat stdout | grep "XFAIL.*:"
cat stdout | grep "XPASS.*:"
cat stdout | grep "ERROR.*:"
- # To check that the testsuite summary is not unduly colorized.
- cat stdout | grep '===='
cat stdout | grep 'test.*expected'
cat stdout | grep 'test.*not run'
+ cat stdout | grep '===='
+ cat stdout | grep '[Ss]ee .*test-suite\.log'
+ cat stdout | grep '[Tt]estsuite summary'
) | grep "$esc" && Exit 1
: For shells with broken 'set -e'
else
--- /dev/null
+#! /usr/bin/env perl
+# Extract the testsuite summary generated by the parallel-tests harness
+# from the output of "make check".
+
+use warnings FATAL => 'all';
+use strict;
+
+my $br = '=' x 76;
+my @sections = ('');
+while (<>)
+ {
+ push @sections, $_, '' if /$br/;
+ $sections[-1] .= $_ if !/$br/;
+ }
+print @sections[1..$#sections-1];
no_test_has_run ()
{
ls -1 *.log | grep -v '^test-suite\.log$' | grep . && Exit 1
- grep ' 0 tests passed' test-suite.log
+ grep '^# TOTAL: *0$' test-suite.log
:
}
&& { cat stdout; Exit 1; }
cat stdout
ls -l
-grep '^2 of 2 .*failed' stdout
+grep '^# TOTAL: *2$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
for x in stdout my.log; do
$FGREP foo.test $x && Exit 1
$FGREP bar.test $x
&& { cat stdout; Exit 1; }
cat stdout
ls -l
-grep '^1 of 2 .*failed' stdout
+grep '^# TOTAL: *2$' stdout
+grep '^# PASS: *1$' stdout
+grep '^# ERROR: *1$' stdout
$FGREP foo.test stdout && Exit 1
$FGREP bar.test stdout
$FGREP baz.test stdout
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Colorized output from the testsuite report shouldn't end up in log files.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+esc='\e'
+
+# Check that grep can parse nonprinting characters.
+# BSD 'grep' works from a pipe, but not a seekable file.
+# GNU or BSD 'grep -a' works on files, but is not portable.
+case `echo "$esc" | $FGREP "$esc"` in
+ "$esc") ;;
+ *) echo "$me: fgrep can't parse nonprinting characters" >&2; Exit 77;;
+esac
+
+TERM=ansi; export TERM
+
+cat >>configure.in <<END
+AC_OUTPUT
+END
+
+cat >Makefile.am <<'END'
+LOG_COMPILER = $(SHELL)
+AUTOMAKE_OPTIONS = color-tests parallel-tests
+TESTS = pass fail skip xpass xfail error
+XFAIL_TESTS = xpass xfail
+END
+
+# FIXME: creative quoting to please maintainer-check.
+echo exit '0' > pass
+echo exit '0' > xpass
+echo exit '1' > fail
+echo exit '1' > xfail
+echo exit '77' > skip
+echo exit '99' > error
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+./configure
+mv config.log config-log # Avoid possible false positives below.
+AM_COLOR_TESTS=always $MAKE -e check && Exit 1
+$FGREP "$esc" *.log && Exit 1
+
+:
grep '^:test-result:FAIL$' bar.log
grep '^:test-result: XPASS$' bar.log
-$EGREP '^(All)? 2 tests? passed' output
-$EGREP 'did not behave|unexpected pass|not run' output && Exit 1
+grep '^# TOTAL: *2$' output
+grep '^# PASS: *2$' output
+for result in FAIL XFAIL XPASS SKIP ERROR; do
+ grep "^# $result: *0$" output
+done
:
test ! -r foo.log || Exit 77
$MAKE test-suite.log >stdout && { cat stdout; Exit 1; }
cat stdout
-grep '^2 of 2 tests failed *$' stdout
-grep '^2 of 2 tests failed\. *$' test-suite.log
+grep '^# ERROR: *2$' stdout
+grep '^# ERROR: *2$' test-suite.log
:
test -f baz.log
test -f bar.log
test -f foo.log
-# The summary should be formatted correctly.
-grep 'failedn' stdout && Exit 1
# clean should remove all log files (but not more).
: > unrelated.log
grep foo.test stdout
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
-grep '2.*tests.*failed' stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
# Now, explicitly retry with all test logs already updated, and ensure
# that the summary is still displayed.
grep foo.test stdout && Exit 1
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
-grep '2.*tests.*failed' stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
# Lazily rerunning only foo should only rerun this one test.
env RECHECK_LOGS=foo.log $MAKE -e check > stdout && { cat stdout; Exit 1; }
grep foo.test stdout
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
-grep '2.*tests.*failed' stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
# Test VERBOSE.
env VERBOSE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; }
cd ..
kill $!
cat parallel/stdout
-test `grep -c PASS parallel/stdout` -eq 8
+test `grep -c '^PASS:' parallel/stdout` -eq 8
# Wait long enough so that there are no open files any more
# when the post-test cleanup runs.
./configure
$MAKE check >stdout || { cat stdout; Exit 1; }
cat stdout
-grep '0 tests' stdout
+for x in TOTAL PASS FAIL XPASS FAIL SKIP ERROR; do
+ grep "^# $x: *0$" stdout
+done
Exit 0
./configure
$MAKE check >stdout && { cat stdout; Exit 1; }
cat stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
$MAKE recheck >stdout && { cat stdout; Exit 1; }
cat stdout
grep foo.test stdout && Exit 1
grep bar.test stdout || Exit 1
grep baz.test stdout || Exit 1
-grep '2 of 2.*failed' stdout
+grep '^# PASS: *0$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
# If we cannot read the log file, then redo it as well.
chmod a-r foo.log
pass-xpass-fail-xfail-skip-error.t
END
+expected_tests=23
expected_pass=10
expected_fail=5
expected_skip=4
grep "^SKIP: $tst\.t, testcase 5" stdout
grep "^ERROR: $tst\.t, testcase 6" stdout
- # Check testsuite summary printed on console.
- sed -e 's/[()]/ /g' -e 's/^/ /' stdout > t
- grep ' 7 of 19 ' t
- grep ' 1 unexpected pass' t
- grep ' 4 test.* not run' t
+ # Check counts of testcases in testsuite summary printed on console.
+ grep "^# TOTAL: *$expected_tests$" stdout
+ grep "^# PASS: *$expected_pass$" stdout
+ grep "^# XPASS: *$expected_xpass$" stdout
+ grep "^# FAIL: *$expected_fail$" stdout
+ grep "^# XFAIL: *$expected_xfail$" stdout
+ grep "^# SKIP: *$expected_skip$" stdout
+ grep "^# ERROR: *$expected_error$" stdout
# Check that the content of, and only of, the test logs with at least
# one failing test case has been copied into `test-suite.log'. Note
test `grep -c '^SKIP:' stdout` -eq 2
test `grep -c '^XFAIL:' stdout` -eq 1
$EGREP '^(FAIL|XPASS|ERROR)' stdout && Exit 1
+ # Check counts of testcases in testsuite summary printed on console.
+ grep "^# TOTAL: *7$" stdout
+ grep "^# PASS: *4$" stdout
+ grep "^# XPASS: *0$" stdout
+ grep "^# FAIL: *0$" stdout
+ grep "^# XFAIL: *1$" stdout
+ grep "^# SKIP: *2$" stdout
+ grep "^# ERROR: *0$" stdout
cd $srcdir
cat output
cat a.log; cat b.log; cat c.log; cat d.log # For debugging.
-$EGREP '^(All)? 3 tests? passed' output
-grep '^(1 test.* not run' output
-$EGREP 'did not behave|unexpected pass' output && Exit 1
+grep '^# TOTAL: *4$' output
+grep '^# PASS: *3$' output
+grep '^# SKIP: *1$' output
+for result in XPASS FAIL XFAIL ERROR; do
+ grep "^# $result: *0$" output
+done
:
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check the testsuite summary with the parallel-tests harness. This
+# script is meant to be sourced by other test script, so that it can
+# be used to check different scenarios (colorized and non-colorized
+# testsuite output, packages with and without bug-report addresses,
+# testsuites in subdirectories, ...)
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+case $use_colors in
+ yes)
+ AM_COLOR_TESTS=always; export AM_COLOR_TESTS
+ TERM=ansi; export TERM
+ am_opts='parallel-tests color-tests'
+ ;;
+ no)
+ am_opts='parallel-tests'
+ ;;
+ *)
+ fatal_ "invalid use_colors='$use_colors'";;
+esac
+
+cat > configure.in <<END
+AC_INIT([GNU AutoFoo], [7.1], [bug-automake@gnu.org])
+AM_INIT_AUTOMAKE([$am_opts])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TEST_EXTENSIONS = .t
+T_LOG_COMPILER = $(SHELL) $(srcdir)/log-compiler
+## Will be defined at runtime.
+TESTS =
+$(TESTS):
+END
+
+cat > log-compiler <<'END'
+#!/bin/sh
+case "$1" in
+ pass*|*/pass*|xpass*|*/xpass*) exit 0;;
+ fail*|*/fail*|xfail*|*/xfail*) exit 1;;
+ skip*|*/skip*) exit 77;;
+ error*|/error*) exit 99;;
+ copy*|*/copy*) cat "$1";;
+ *) exit 99;;
+esac
+END
+
+# Quite complexish, but allow the tests in client scripts to be written
+# in a "data-driven fashion".
+do_check ()
+{
+ cat > summary.exp
+ expect_failure=false
+ xfail_tests=''
+ tests="TESTS='$*'"
+ for t in $*; do
+ case $t in fail*|xpass*|error*) expect_failure=:;; esac
+ case $t in xfail*|xpass*) xfail_tests="$xfail_tests $t";; esac
+ done
+ test -z "$xfail_tests" || xfail_tests="XFAIL_TESTS='$xfail_tests'"
+ st=0
+ eval "env $tests $xfail_tests \$MAKE -e check > stdout || st=\$?"
+ cat stdout
+ if $expect_failure; then
+ test $st -gt 0 || Exit 1
+ else
+ test $st -eq 0 || Exit 1
+ fi
+ $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \
+ || fatal_ "cannot extract testsuite summary"
+ cat summary.exp
+ cat summary.got
+ if test $use_colors = yes; then
+ # Use cmp, not diff, because the files might contain binary data.
+ compare=cmp
+ else
+ compare=diff
+ fi
+ $compare summary.exp summary.got || Exit 1
+}
+
+br='============================================================================'
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+:
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check coloring of the testsuite summary.
+
+. ./defs-static || Exit 1
+
+use_colors=yes
+use_vpath=no
+
+. "$testsrcdir"/testsuite-summary-checks.sh || Exit 99
+
+./configure
+
+# ANSI colors.
+red='\e[0;31m'
+grn='\e[0;32m'
+lgn='\e[1;32m'
+blu='\e[1;34m'
+mgn='\e[0;35m'
+brg='\e[1m';
+std='\e[m';
+
+success_header="\
+${grn}${br}${std}
+${grn}Testsuite summary for GNU AutoFoo 7.1${std}
+${grn}${br}${std}"
+
+success_footer=${grn}${br}${std}
+
+failure_header="\
+${red}${br}${std}
+${red}Testsuite summary for GNU AutoFoo 7.1${std}
+${red}${br}${std}"
+
+failure_footer="\
+${red}${br}${std}
+${red}See ./test-suite.log${std}
+${red}Please report to bug-automake@gnu.org${std}
+${red}${br}${std}"
+
+do_check '' <<END
+$success_header
+# TOTAL: 0
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t <<END
+$success_header
+${brg}# TOTAL: 1${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check skip.t <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check xfail.t <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check fail.t <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+do_check xpass.t <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+do_check error.t <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+do_check pass.t xfail.t skip.t <<END
+$success_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t fail.t skip.t <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+do_check pass.t xfail.t xpass.t <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+do_check skip.t xfail.t error.t <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+do_check pass.t skip.t xfail.t fail.t xpass.t error.t <<END
+$failure_header
+${brg}# TOTAL: 6${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+${red}# FAIL: 1${std}
+${red}# XPASS: 1${std}
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+:
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check test counts in the testsuite summary, with test drivers allowing
+# multiple test results per test script, and for a huge number of tests.
+# Incidentally, this test also checks that the testsuite summary doesn't
+# give any bug-report address if it's not defined.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+for s in trivial-test-driver extract-testsuite-summary; do
+ cp "$testsrcdir/$s" . || fatal_ "failed to fetch auxiliary script $s"
+done
+
+br='============================================================================'
+
+header="\
+${br}
+Testsuite summary for $me 1.0
+${br}"
+
+footer="\
+${br}
+See ./test-suite.log
+${br}"
+
+echo AC_OUTPUT >> configure.in
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
+TESTS = all.test
+# Without this, the test driver will be horrendously slow.
+END
+
+cat > all.test <<'END'
+#!/bin/sh
+cat results.txt || { echo ERROR: weird; exit 99; }
+END
+chmod a+x all.test
+
+$PERL -w -e '
+ use warnings FATAL => "all";
+ use strict;
+
+ # FIXME: we would like this to be 1000 or even 10000, but the current
+ # implementation is too slow to handle that :-(
+ my $base = 5;
+ my %count = (
+ TOTAL => $base * 1000,
+ PASS => $base * 700,
+ SKIP => $base * 200,
+ XFAIL => $base * 80,
+ FAIL => $base * 10,
+ XPASS => $base * 7,
+ ERROR => $base * 3,
+ );
+ my @results = qw/PASS SKIP XFAIL FAIL XPASS ERROR/;
+
+ open (RES, ">results.txt") or die "opening results.txt: $!\n";
+ open (CNT, ">count.txt") or die "opening count.txt: $!\n";
+
+ printf CNT "# %-6s %d\n", "TOTAL:", $count{TOTAL};
+ for my $res (@results)
+ {
+ my $uc_res = uc $res;
+ print STDERR "Generating list of $res ...\n";
+ for (1..$count{$res})
+ {
+ print RES "$uc_res: $_\n";
+ }
+ printf CNT "# %-6s %d\n", $res . ":", $count{$res};
+ }
+'
+
+(echo "$header" && cat count.txt && echo "$footer") > summary.exp
+
+$ACLOCAL
+$AUTOMAKE -a
+$AUTOCONF
+
+./configure
+
+($MAKE check || : > make.fail) | tee stdout
+test -f make.fail
+
+$PERL -w extract-testsuite-summary stdout > summary.got
+cat summary.exp
+cat summary.got
+diff summary.exp summary.got || Exit 1
+
+:
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check test counts in the testsuite summary.
+
+. ./defs-static || Exit 1
+
+use_colors=no
+use_vpath=no
+
+. "$testsrcdir"/testsuite-summary-checks.sh || Exit 99
+
+seq_ ()
+{
+ case $# in
+ 2) l=$1 u=$2;;
+ *) fatal_ "incorrect usage of 'seq_' function";;
+ esac
+ seq $1 $2 || {
+ i=$l
+ while test $i -le $u; do
+ echo $i
+ i=`expr $i + 1`
+ done
+ }
+}
+
+./configure
+
+header="\
+${br}
+Testsuite summary for GNU AutoFoo 7.1
+${br}"
+
+success_footer=${br}
+
+failure_footer="\
+${br}
+See ./test-suite.log
+Please report to bug-automake@gnu.org
+${br}"
+
+# Corner cases.
+
+do_check '' <<END
+$header
+# TOTAL: 0
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t <<END
+$header
+# TOTAL: 1
+# PASS: 1
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check fail.t <<END
+$header
+# TOTAL: 1
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 1
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# Some simpler checks, with low or moderate number of tests.
+
+do_check skip.t skip2.t skip3.t xfail.t xfail2.t <<END
+$header
+# TOTAL: 5
+# PASS: 0
+# SKIP: 3
+# XFAIL: 2
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t pass2.t xfail.t xpass.t error.t error2.t <<END
+$header
+# TOTAL: 6
+# PASS: 2
+# SKIP: 0
+# XFAIL: 1
+# FAIL: 0
+# XPASS: 1
+# ERROR: 2
+$failure_footer
+END
+
+pass_count=22
+skip_count=19
+xfail_count=21
+fail_count=18
+xpass_count=23
+error_count=17
+tests_count=120
+
+pass=` seq_ 1 $pass_count | sed 's/.*/pass-&.t/'`
+skip=` seq_ 1 $skip_count | sed 's/.*/skip-&.t/'`
+xfail=`seq_ 1 $xfail_count | sed 's/.*/xfail-&.t/'`
+fail=` seq_ 1 $fail_count | sed 's/.*/fail-&.t/'`
+xpass=`seq_ 1 $xpass_count | sed 's/.*/xpass-&.t/'`
+error=`seq_ 1 $error_count | sed 's/.*/error-&.t/'`
+
+do_check $pass $skip $xfail $fail $xpass $error <<END
+$header
+# TOTAL: $tests_count
+# PASS: $pass_count
+# SKIP: $skip_count
+# XFAIL: $xfail_count
+# FAIL: $fail_count
+# XPASS: $xpass_count
+# ERROR: $error_count
+$failure_footer
+END
+
+# Mild stress test with a lot of test scripts.
+
+tests_count=1888
+pass_count=1403
+skip_count=292
+xfail_count=41
+fail_count=126
+xpass_count=17
+error_count=9
+
+pass=` seq_ 1 $pass_count | sed 's/.*/pass-&.t/'`
+skip=` seq_ 1 $skip_count | sed 's/.*/skip-&.t/'`
+xfail=`seq_ 1 $xfail_count | sed 's/.*/xfail-&.t/'`
+fail=` seq_ 1 $fail_count | sed 's/.*/fail-&.t/'`
+xpass=`seq_ 1 $xpass_count | sed 's/.*/xpass-&.t/'`
+error=`seq_ 1 $error_count | sed 's/.*/error-&.t/'`
+
+do_check $pass $skip $xfail $fail $xpass $error <<END
+$header
+# TOTAL: $tests_count
+# PASS: $pass_count
+# SKIP: $skip_count
+# XFAIL: $xfail_count
+# FAIL: $fail_count
+# XPASS: $xpass_count
+# ERROR: $error_count
+$failure_footer
+END
+
+:
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that the global testsuite log file referenced in the testsuite
+# summary and in the global testsuite log itself is correct.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+mv configure.in configure.stub
+
+cat > fail << 'END'
+#!/bin/sh
+exit 1
+END
+chmod a+x fail
+
+cat configure.stub - > configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_SUITE_LOG = my_test_suite.log
+TESTS = fail
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+mkdir build
+cd build
+
+../configure
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See \./my_test_suite\.log$' stdout
+
+mkdir bar
+TEST_SUITE_LOG=bar/bar.log $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See \./bar/bar\.log$' stdout
+
+cd ..
+
+echo SUBDIRS = sub > Makefile.am
+mkdir sub
+echo TESTS = fail > sub/Makefile.am
+mv fail sub
+
+cat configure.stub - > configure.in <<'END'
+AC_CONFIG_FILES([sub/Makefile])
+AC_OUTPUT
+END
+
+$ACLOCAL --force
+$AUTOCONF --force
+$AUTOMAKE
+
+./configure
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See sub/test-suite\.log$' stdout
+cd sub
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See sub/test-suite\.log$' stdout
+cd ..
+
+TEST_SUITE_LOG=foo.log $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See sub/foo\.log$' stdout
+
+:
## Run the test script, get test cases results, display them on console.
-tmp_out=$log_file-out.tmp
-tmp_res=$log_file-res.tmp
+tmp_output=$log_file-output.tmp
+tmp_results=$log_file-results.tmp
+tmp_status=$log_file-status.tmp
-"$@" 2>&1 | tee $tmp_out | (
+"$@" 2>&1 | tee $tmp_output | (
i=0 st=0
- : > $tmp_res
+ exec 5> $tmp_results
+ : > $tmp_status
while read line; do
+ result=
case $line in
- PASS:*|FAIL:*|XPASS:*|XFAIL:*|SKIP:*|ERROR:*)
- i=`expr $i + 1`
- result=`LC_ALL=C expr "$line" : '\([A-Z]*\):.*'`
- case $result in FAIL|XPASS|ERROR) st=1;; esac
- # Output testcase result to console.
- echo "$result: $test_name, testcase $i"
- # Register testcase outcome for the log file.
- echo ":test-result: $line" >> $tmp_res
- echo >> $tmp_res
- ;;
+ PASS:*) result=PASS ;;
+ FAIL:*) result=FAIL ;;
+ XPASS:*) result=XPASS ;;
+ XFAIL:*) result=XFAIL ;;
+ SKIP:*) result=SKIP ;;
+ ERROR:*) result=ERROR ;;
esac
+ if test -n "$result"; then
+ case $result in FAIL|XPASS|ERROR) st=1;; esac
+ # Output testcase result to console.
+ echo "$result: $test_name"
+ # Register testcase outcome for the log file.
+ echo ":test-result: $line" >&5
+ echo >&5
+ fi
done
- exit $st
-)
+ test $st -eq 0 || echo fail > $tmp_status
+) | awk '{ print $0 ", testcase " NR }'
-if test $? -eq 0; then
+if test ! -s $tmp_status; then
global_result=PASS
else
global_result=FAIL
echo "$global_result: $test_name"
echo "$global_result: $test_name" | sed 's/./=/g'
echo
- cat $tmp_res
+ cat $tmp_results
echo
echo --------------------
echo
- cat $tmp_out
+ cat $tmp_output
} > $log_file
-rm -f $tmp_out $tmp_res
+rm -f $tmp_output $tmp_results $tmp_status
## And we're done.
projects / thirdparty / automake.git / commitdiff
