From eaac33fe13811cf104f45a09a6ece911c135fa83 Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Fri, 1 Jul 2011 11:12:31 +0200 Subject: [PATCH] parallel-tests: simplify testsuite summary Prefer a more deterministic, "tabular" format for the testsuite summary, always listing the numbers of passed, failed, xfailed, xpassed, skipped and errored tests, even when these numbers are zero. This simplify the logic of testsuite summary creation, makes it more easily machine-parseable, and will probably allow for easier addition of new kinds of test results in the future. * lib/am/check.am (am__tty_colors_dummy): New make variable, to reduce code duplication. Extracted from previous versions of $(am__tty_colors), and extended by defining two new variables `$mgn' and `$brg'. [%?COLOR%, %!?COLOR%] (am__tty_colors): Use that new variable. (am__text_box): Delete, is not needed anymore. ($(TEST_SUITE_LOG)): Rewrite associated rules to implement the new testsuite summary format. * NEWS: Update. * tests/check10.test: Don't run with the parallel-tests harness too, that makes no sense anymore. * tests/color.test: Update and adjust. * tests/color2.test: Likewise. * tests/parallel-tests.test: Likewise. * tests/parallel-tests3.test: Likewise. * tests/parallel-tests6.test: Likewise. * tests/parallel-tests9.test: Likewise. * tests/parallel-tests-unreadable-log.test: Likewise. * tests/parallel-tests-empty-testlogs.test: Likewise. * tests/parallel-tests-log-override-recheck.test: Likewise. * tests/parallel-tests-no-spurious-summary.test: Likewise. * tests/test-driver-custom-multitest.test: Likewise. * tests/test-driver-end-test-results.test: Likewise. * tests/parallel-tests-no-color-in-log.test: New test. * tests/testsuite-summary-color.test: Likewise. * tests/testsuite-summary-count.test: Likewise. * tests/testsuite-summary-count-many.test: Likewise. * tests/testsuite-summary-reference-log.test: Likewise. * tests/testsuite-summary-checks.sh: New auxiliary script, used by the new tests above. * tests/extract-testsuite-summary: Likewise. * tests/trivial-test-driver: Optimize for speed when there are lots of of tests. * tests/Makefile.am (EXTRA_DIST): Distribute them. (testsuite-summary-color.log, testsuite-summary-count.log): Depend on them. (testsuite-summary-count-many.log): Depend on the auxiliary scripts 'trivial-test-driver' and 'extract-testsuite-summary'. (TESTS): Update. --- ChangeLog | 49 +++++ NEWS | 6 + lib/Automake/tests/Makefile.in | 125 +++++------ lib/am/check.am | 147 ++++++------- tests/Makefile.am | 12 ++ tests/Makefile.in | 150 ++++++------- tests/check10.test | 2 + tests/color.test | 8 +- tests/color2.test | 8 +- tests/extract-testsuite-summary | 15 ++ tests/parallel-tests-empty-testlogs.test | 2 +- .../parallel-tests-log-override-recheck.test | 8 +- tests/parallel-tests-no-color-in-log.test | 62 ++++++ tests/parallel-tests-no-spurious-summary.test | 7 +- tests/parallel-tests-unreadable-log.test | 4 +- tests/parallel-tests.test | 14 +- tests/parallel-tests3.test | 2 +- tests/parallel-tests6.test | 4 +- tests/parallel-tests9.test | 7 +- tests/test-driver-custom-multitest.test | 22 +- tests/test-driver-end-test-results.test | 9 +- tests/testsuite-summary-checks.sh | 106 ++++++++++ tests/testsuite-summary-color.test | 199 ++++++++++++++++++ tests/testsuite-summary-count-many.test | 105 +++++++++ tests/testsuite-summary-count.test | 176 ++++++++++++++++ tests/testsuite-summary-reference-log.test | 88 ++++++++ tests/trivial-test-driver | 47 +++-- 27 files changed, 1122 insertions(+), 262 deletions(-) create mode 100644 tests/extract-testsuite-summary create mode 100755 tests/parallel-tests-no-color-in-log.test create mode 100755 tests/testsuite-summary-checks.sh create mode 100755 tests/testsuite-summary-color.test create mode 100755 tests/testsuite-summary-count-many.test create mode 100755 tests/testsuite-summary-count.test create mode 100755 tests/testsuite-summary-reference-log.test diff --git a/ChangeLog b/ChangeLog index 6a062e324..cc64b7760 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,52 @@ +2011-07-07 Stefano Lattarini + + parallel-tests: simplify testsuite summary + Prefer a more deterministic, "tabular" format for the testsuite + summary, always listing the numbers of passed, failed, xfailed, + xpassed, skipped and errored tests, even when these numbers are + zero. This simplify the logic of testsuite summary creation, + makes it more easily machine-parseable, and will probably allow + for easier addition of new kinds of test results in the future. + * lib/am/check.am (am__tty_colors_dummy): New make variable, to + reduce code duplication. Extracted from previous versions of + $(am__tty_colors), and extended by defining two new variables + `$mgn' and `$brg'. + [%?COLOR%, %!?COLOR%] (am__tty_colors): Use that new variable. + (am__text_box): Delete, is not needed anymore. + ($(TEST_SUITE_LOG)): Rewrite associated rules to implement the + new testsuite summary format. + * NEWS: Update. + * tests/check10.test: Don't run with the parallel-tests harness + too, that makes no sense anymore. + * tests/color.test: Update and adjust. + * tests/color2.test: Likewise. + * tests/parallel-tests.test: Likewise. + * tests/parallel-tests3.test: Likewise. + * tests/parallel-tests6.test: Likewise. + * tests/parallel-tests9.test: Likewise. + * tests/parallel-tests-unreadable-log.test: Likewise. + * tests/parallel-tests-empty-testlogs.test: Likewise. + * tests/parallel-tests-log-override-recheck.test: Likewise. + * tests/parallel-tests-no-spurious-summary.test: Likewise. + * tests/test-driver-custom-multitest.test: Likewise. + * tests/test-driver-end-test-results.test: Likewise. + * tests/parallel-tests-no-color-in-log.test: New test. + * tests/testsuite-summary-color.test: Likewise. + * tests/testsuite-summary-count.test: Likewise. + * tests/testsuite-summary-count-many.test: Likewise. + * tests/testsuite-summary-reference-log.test: Likewise. + * tests/testsuite-summary-checks.sh: New auxiliary script, used + by the new tests above. + * tests/extract-testsuite-summary: Likewise. + * tests/trivial-test-driver: Optimize for speed when there are + lots of of tests. + * tests/Makefile.am (EXTRA_DIST): Distribute them. + (testsuite-summary-color.log, testsuite-summary-count.log): Depend + on them. + (testsuite-summary-count-many.log): Depend on the auxiliary scripts + 'trivial-test-driver' and 'extract-testsuite-summary'. + (TESTS): Update. + 2011-07-07 Stefano Lattarini parallel-tests: new recognized test result 'ERROR' diff --git a/NEWS b/NEWS index a27b886bb..461429754 100644 --- a/NEWS +++ b/NEWS @@ -21,6 +21,12 @@ New in 1.11a: as failures even when the test originating them was listed in XFAIL_TESTS). + - The testsuite summary displayed by the parallel-test harness has a + completely new format, that always listing the numbers of passed, + failed, xfailed, xpassed, skipped and errored tests, even when these + numbers are zero (but using smart coloring when the color-tests option + is in effect). + - The default testsuite driver offered by the 'parallel-tests' option is now implemented (partly at least) with the help of automake-provided auxiliary scripts (e.g., `test-driver'), instead of relying entirely diff --git a/lib/Automake/tests/Makefile.in b/lib/Automake/tests/Makefile.in index 82ef9e1d8..6139c8bed 100644 --- a/lib/Automake/tests/Makefile.in +++ b/lib/Automake/tests/Makefile.in @@ -71,12 +71,14 @@ am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) am__v_at_0 = @ SOURCES = DIST_SOURCES = +am__tty_colors_dummy = \ + mgn= red= grn= lgn= blu= brg= std=; \ + am__color_tests=no # If stdout is a non-dumb tty, use colors. If test -t is not supported, # then this fails; a conservative approach. Of course do not redirect # stdout here, just stderr. am__tty_colors = \ -am__color_tests=no; \ -red=; grn=; lgn=; blu=; std=; \ +$(am__tty_colors_dummy); \ test "X$(AM_COLOR_TESTS)" != Xno \ && test "X$$TERM" != Xdumb \ && { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \ @@ -86,6 +88,8 @@ test "X$(AM_COLOR_TESTS)" != Xno \ grn=''; \ lgn=''; \ blu=''; \ + mgn=''; \ + brg=''; \ std=''; \ } am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; @@ -111,23 +115,6 @@ am__base_list = \ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' # Restructured Text title. am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//' -# Put stdin (possibly several lines separated by ". ") in a box. -# Prefix each line by 'col' and terminate each with 'std', for coloring. -# Multi line coloring is problematic with "less -R", so we really need -# to color each line individually. -am__text_box = $(AWK) '{ \ - n = split($$0, lines, "\\. "); max = 0; \ - for (i = 1; i <= n; ++i) \ - if (max < length(lines[i])) \ - max = length(lines[i]); \ - for (i = 0; i < max; ++i) \ - line = line "="; \ - print col line std; \ - for (i = 1; i <= n; ++i) \ - if (lines[i]) \ - print col lines[i] std; \ - print col line std; \ -}' # Solaris 10 'make', and several other traditional 'make' implementations, # pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it # by disabling -e (using the XSI extension "set +e") if it's set. @@ -343,7 +330,7 @@ cscope cscopelist: $(TEST_SUITE_LOG): $(TEST_LOGS) - @$(am__sh_e_setup); \ + @$(am__sh_e_setup); $(am__tty_colors); \ rst_magic=":test-result:"; \ nlinit=`echo 'nl="'; echo '"'`; eval "$$nlinit"; unset nlinit; \ list='$(TEST_LOGS)'; \ @@ -362,7 +349,7 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) exec 0<&5; \ done; \ exec 5<&-; \ - results=`echo "$$results1" && echo "$$results2"` \ + results=`echo "$$results1" && echo "$$results2"`; \ all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \ fail=`echo "$$results" | grep -c '^FAIL'`; \ pass=`echo "$$results" | grep -c '^PASS'`; \ @@ -370,44 +357,44 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) xfail=`echo "$$results" | grep -c '^XFAIL'`; \ xpass=`echo "$$results" | grep -c '^XPASS'`; \ error=`echo "$$results" | grep -c '^ERROR'`; \ - fail=`expr $$fail + $$error`; \ - failures=`expr $$fail + $$xpass`; \ - all=`expr $$all - $$skip`; \ - if test "$$all" -eq 1; then tests=test; All=; \ - else tests=tests; All="All "; fi; \ - case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \ - fail=0:xpass=0:xfail=0) \ - msg="$$All$$all $$tests passed. "; \ - exit=true;; \ - fail=0:xpass=0:xfail=*) \ - msg="$$All$$all $$tests behaved as expected"; \ - if test "$$xfail" -eq 1; then xfailures=failure; \ - else xfailures=failures; fi; \ - msg="$$msg ($$xfail expected $$xfailures). "; \ - exit=true;; \ - fail=*:xpass=0:xfail=*) \ - msg="$$fail of $$all $$tests failed. "; \ - exit=false;; \ - fail=*:xpass=*:xfail=*) \ - msg="$$failures of $$all $$tests did not behave as expected"; \ - if test "$$xpass" -eq 1; then xpasses=pass; \ - else xpasses=passes; fi; \ - msg="$$msg ($$xpass unexpected $$xpasses). "; \ - exit=false;; \ - *) \ - echo >&2 "incorrect case"; exit 4;; \ - esac; \ - if test "$$skip" -ne 0; then \ - if test "$$skip" -eq 1; then \ - msg="$$msg($$skip test was not run). "; \ - else \ - msg="$$msg($$skip tests were not run). "; \ - fi; \ - fi; \ + if test `expr $$fail + $$xpass + $$error` -eq 0; then \ + success=true; \ + else \ + success=false; \ + fi; \ + br='==================='; br=$$br$$br$$br$$br; \ + result_count () \ + { \ + if test x"$$1" = x"--maybe-color"; then \ + maybe_colorize=yes; \ + elif test x"$$1" = x"--no-color"; then \ + maybe_colorize=no; \ + else \ + echo "$@: invalid 'result_count' usage" >&2; exit 4; \ + fi; \ + shift; \ + desc=$$1 count=$$2; \ + if test $$maybe_colorize = yes && test $$count -gt 0; then \ + color_start=$$3 color_end=$$std; \ + else \ + color_start= color_end=; \ + fi; \ + echo "$${color_start}# $$desc $$count$${color_end}"; \ + }; \ + create_testsuite_report () \ + { \ + result_count $$1 "TOTAL:" $$all "$$brg"; \ + result_count $$1 "PASS: " $$pass "$$grn"; \ + result_count $$1 "SKIP: " $$skip "$$blu"; \ + result_count $$1 "XFAIL:" $$xfail "$$lgn"; \ + result_count $$1 "FAIL: " $$fail "$$red"; \ + result_count $$1 "XPASS:" $$xpass "$$red"; \ + result_count $$1 "ERROR:" $$error "$$mgn"; \ + }; \ { \ echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \ $(am__rst_title); \ - echo "$$msg"; \ + create_testsuite_report --no-color; \ echo; \ echo ".. contents:: :depth: 2"; \ echo; \ @@ -420,21 +407,25 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) done; \ } >$(TEST_SUITE_LOG).tmp; \ mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ - if test "$$failures" -ne 0; then \ - msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \ - if test -n "$(PACKAGE_BUGREPORT)"; then \ - msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \ - fi; \ - fi; \ - test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \ - $(am__tty_colors); \ - if $$exit; then \ + if $$success; then \ col="$$grn"; \ else \ col="$$red"; \ + test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \ + fi; \ + echo "$${col}$$br$${std}"; \ + echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \ + echo "$${col}$$br$${std}"; \ + create_testsuite_report --maybe-color; \ + echo "$$col$$br$$std"; \ + if $$success; then :; else \ + echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \ + if test -n "$(PACKAGE_BUGREPORT)"; then \ + echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \ + fi; \ + echo "$$col$$br$$std"; \ fi; \ - echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \ - $$exit + $$success # Run all the tests. check-TESTS: diff --git a/lib/am/check.am b/lib/am/check.am index 3cedd4635..73306bce3 100644 --- a/lib/am/check.am +++ b/lib/am/check.am @@ -15,13 +15,15 @@ ## You should have received a copy of the GNU General Public License ## along with this program. If not, see . +am__tty_colors_dummy = \ + mgn= red= grn= lgn= blu= brg= std=; \ + am__color_tests=no if %?COLOR% # If stdout is a non-dumb tty, use colors. If test -t is not supported, # then this fails; a conservative approach. Of course do not redirect # stdout here, just stderr. am__tty_colors = \ -am__color_tests=no; \ -red=; grn=; lgn=; blu=; std=; \ +$(am__tty_colors_dummy); \ test "X$(AM_COLOR_TESTS)" != Xno \ && test "X$$TERM" != Xdumb \ && { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \ @@ -31,10 +33,12 @@ test "X$(AM_COLOR_TESTS)" != Xno \ grn=''; \ lgn=''; \ blu=''; \ + mgn=''; \ + brg=''; \ std=''; \ } else !%?COLOR% -am__tty_colors = red= grn= lgn= blu= std= am__color_tests=no +am__tty_colors = $(am__tty_colors_dummy) endif !%?COLOR% .PHONY: check-TESTS @@ -74,24 +78,6 @@ include inst-vars.am # Restructured Text title. am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//' -# Put stdin (possibly several lines separated by ". ") in a box. -# Prefix each line by 'col' and terminate each with 'std', for coloring. -# Multi line coloring is problematic with "less -R", so we really need -# to color each line individually. -am__text_box = $(AWK) '{ \ - n = split($$0, lines, "\\. "); max = 0; \ - for (i = 1; i <= n; ++i) \ - if (max < length(lines[i])) \ - max = length(lines[i]); \ - for (i = 0; i < max; ++i) \ - line = line "="; \ - print col line std; \ - for (i = 1; i <= n; ++i) \ - if (lines[i]) \ - print col lines[i] std; \ - print col line std; \ -}' - # Solaris 10 'make', and several other traditional 'make' implementations, # pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it # by disabling -e (using the XSI extension "set +e") if it's set. @@ -139,7 +125,7 @@ esac; \ $(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT) $(TEST_SUITE_LOG): $(TEST_LOGS) - @$(am__sh_e_setup); \ + @$(am__sh_e_setup); $(am__tty_colors); \ ## The custom reStructuredText field used to register the outcome of ## a test. This is for supporting test protocols, such as TAP and ## SubUnit, where a single test script can run multiple tests, each @@ -179,8 +165,8 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) done; \ ## We don't need this anymore. exec 5<&-; \ -## Prepare the test suite summary. - results=`echo "$$results1" && echo "$$results2"` \ +## Prepare data for the test suite summary. + results=`echo "$$results1" && echo "$$results2"`; \ all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \ fail=`echo "$$results" | grep -c '^FAIL'`; \ pass=`echo "$$results" | grep -c '^PASS'`; \ @@ -188,47 +174,54 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) xfail=`echo "$$results" | grep -c '^XFAIL'`; \ xpass=`echo "$$results" | grep -c '^XPASS'`; \ error=`echo "$$results" | grep -c '^ERROR'`; \ -## FIXME: for the moment, we count errors as failures, otherwise the code -## that displays the testsuite summary will become too complicated. - fail=`expr $$fail + $$error`; \ - failures=`expr $$fail + $$xpass`; \ - all=`expr $$all - $$skip`; \ - if test "$$all" -eq 1; then tests=test; All=; \ - else tests=tests; All="All "; fi; \ - case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \ - fail=0:xpass=0:xfail=0) \ - msg="$$All$$all $$tests passed. "; \ - exit=true;; \ - fail=0:xpass=0:xfail=*) \ - msg="$$All$$all $$tests behaved as expected"; \ - if test "$$xfail" -eq 1; then xfailures=failure; \ - else xfailures=failures; fi; \ - msg="$$msg ($$xfail expected $$xfailures). "; \ - exit=true;; \ - fail=*:xpass=0:xfail=*) \ - msg="$$fail of $$all $$tests failed. "; \ - exit=false;; \ - fail=*:xpass=*:xfail=*) \ - msg="$$failures of $$all $$tests did not behave as expected"; \ - if test "$$xpass" -eq 1; then xpasses=pass; \ - else xpasses=passes; fi; \ - msg="$$msg ($$xpass unexpected $$xpasses). "; \ - exit=false;; \ - *) \ - echo >&2 "incorrect case"; exit 4;; \ - esac; \ - if test "$$skip" -ne 0; then \ - if test "$$skip" -eq 1; then \ - msg="$$msg($$skip test was not run). "; \ - else \ - msg="$$msg($$skip tests were not run). "; \ - fi; \ - fi; \ +## Whether the testsuite was successful or not. + if test `expr $$fail + $$xpass + $$error` -eq 0; then \ + success=true; \ + else \ + success=false; \ + fi; \ +## Make $br a line of exactly 76 `=' characters, that will be used to +## enclose the testsuite summary report when displayed on the console. + br='==================='; br=$$br$$br$$br$$br; \ +## When writing the test summary to the console, we want to color a line +## reporting the count of some result *only* if at least one test +## experienced such a result. This function is handy in this regard. + result_count () \ + { \ + if test x"$$1" = x"--maybe-color"; then \ + maybe_colorize=yes; \ + elif test x"$$1" = x"--no-color"; then \ + maybe_colorize=no; \ + else \ + echo "$@: invalid 'result_count' usage" >&2; exit 4; \ + fi; \ + shift; \ + desc=$$1 count=$$2; \ + if test $$maybe_colorize = yes && test $$count -gt 0; then \ + color_start=$$3 color_end=$$std; \ + else \ + color_start= color_end=; \ + fi; \ + echo "$${color_start}# $$desc $$count$${color_end}"; \ + }; \ +## A shell function that creates the testsuite summary. We need it +## because we have to create *two* summaries, one for test-suite.log, +## and a possibly-colorized one for console output. + create_testsuite_report () \ + { \ + result_count $$1 "TOTAL:" $$all "$$brg"; \ + result_count $$1 "PASS: " $$pass "$$grn"; \ + result_count $$1 "SKIP: " $$skip "$$blu"; \ + result_count $$1 "XFAIL:" $$xfail "$$lgn"; \ + result_count $$1 "FAIL: " $$fail "$$red"; \ + result_count $$1 "XPASS:" $$xpass "$$red"; \ + result_count $$1 "ERROR:" $$error "$$mgn"; \ + }; \ ## Write "global" testsuite log. { \ echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \ $(am__rst_title); \ - echo "$$msg"; \ + create_testsuite_report --no-color; \ echo; \ echo ".. contents:: :depth: 2"; \ echo; \ @@ -243,22 +236,30 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) done; \ } >$(TEST_SUITE_LOG).tmp; \ mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ - if test "$$failures" -ne 0; then \ - msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \ - if test -n "$(PACKAGE_BUGREPORT)"; then \ - msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \ - fi; \ - fi; \ - test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \ -## Emit the test summary on the console, and exit. - $(am__tty_colors); \ - if $$exit; then \ +## Emit the test summary on the console. + if $$success; then \ col="$$grn"; \ else \ col="$$red"; \ + test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \ + fi; \ +## Multi line coloring is problematic with "less -R", so we really need +## to color each line individually. + echo "$${col}$$br$${std}"; \ + echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \ + echo "$${col}$$br$${std}"; \ +## This is expected to go to the console, so it might have to be colorized. + create_testsuite_report --maybe-color; \ + echo "$$col$$br$$std"; \ + if $$success; then :; else \ + echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \ + if test -n "$(PACKAGE_BUGREPORT)"; then \ + echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \ + fi; \ + echo "$$col$$br$$std"; \ fi; \ - echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \ - $$exit +## Be sure to exit with the proper exit status. + $$success RECHECK_LOGS = $(TEST_LOGS) diff --git a/tests/Makefile.am b/tests/Makefile.am index 64299b8f4..0d41a1c1e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -733,6 +733,10 @@ parallel-test-driver-install.test \ parallel-tests-no-spurious-summary.test \ parallel-tests-exit-statuses.test \ parallel-tests-console-output.test \ +testsuite-summary-color.test \ +testsuite-summary-count.test \ +testsuite-summary-count-many.test \ +testsuite-summary-reference-log.test \ test-driver-end-test-results.test \ test-driver-custom-no-extra-driver.test \ test-driver-custom.test \ @@ -1095,6 +1099,14 @@ test-driver-custom-multitest-recheck2.log: trivial-test-driver test-driver-custom-html.log: trivial-test-driver EXTRA_DIST += trivial-test-driver +testsuite-summary-color.log testsuite-summary-count.log: \ + testsuite-summary-checks.sh extract-testsuite-summary +EXTRA_DIST += testsuite-summary-checks.sh +EXTRA_DIST += extract-testsuite-summary + +testsuite-summary-count-many.log: trivial-test-driver +testsuite-summary-count-many.log: extract-testsuite-summary + # Dependencies valid for each test case. $(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION) diff --git a/tests/Makefile.in b/tests/Makefile.in index 5ef82da48..c72b1cf25 100644 --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -76,12 +76,14 @@ am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) am__v_at_0 = @ SOURCES = DIST_SOURCES = +am__tty_colors_dummy = \ + mgn= red= grn= lgn= blu= brg= std=; \ + am__color_tests=no # If stdout is a non-dumb tty, use colors. If test -t is not supported, # then this fails; a conservative approach. Of course do not redirect # stdout here, just stderr. am__tty_colors = \ -am__color_tests=no; \ -red=; grn=; lgn=; blu=; std=; \ +$(am__tty_colors_dummy); \ test "X$(AM_COLOR_TESTS)" != Xno \ && test "X$$TERM" != Xdumb \ && { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \ @@ -91,6 +93,8 @@ test "X$(AM_COLOR_TESTS)" != Xno \ grn=''; \ lgn=''; \ blu=''; \ + mgn=''; \ + brg=''; \ std=''; \ } am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; @@ -116,23 +120,6 @@ am__base_list = \ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' # Restructured Text title. am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//' -# Put stdin (possibly several lines separated by ". ") in a box. -# Prefix each line by 'col' and terminate each with 'std', for coloring. -# Multi line coloring is problematic with "less -R", so we really need -# to color each line individually. -am__text_box = $(AWK) '{ \ - n = split($$0, lines, "\\. "); max = 0; \ - for (i = 1; i <= n; ++i) \ - if (max < length(lines[i])) \ - max = length(lines[i]); \ - for (i = 0; i < max; ++i) \ - line = line "="; \ - print col line std; \ - for (i = 1; i <= n; ++i) \ - if (lines[i]) \ - print col lines[i] std; \ - print col line std; \ -}' # Solaris 10 'make', and several other traditional 'make' implementations, # pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it # by disabling -e (using the XSI extension "set +e") if it's set. @@ -282,19 +269,19 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ MAINTAINERCLEANFILES = $(parallel_tests) $(instspc_tests) EXTRA_DIST = ChangeLog-old gen-parallel-tests instspc-tests.sh \ - $(TESTS) trivial-test-driver + $(TESTS) trivial-test-driver testsuite-summary-checks.sh \ + extract-testsuite-summary XFAIL_TESTS = all.test auxdir2.test cond17.test gcj6.test \ override-conditional-2.test pr8365-remake-timing.test \ yacc-dist-nobuild-subdir.test txinfo5.test \ $(instspc_xfail_tests) parallel_tests = backcompat5-p.test check-exported-srcdir-p.test \ check-tests-in-builddir-p.test check-tests_environment-p.test \ - check-p.test check10-p.test check11-p.test check12-p.test \ - check2-p.test check3-p.test check4-p.test check5-p.test \ - check6-p.test check7-p.test check8-p.test check9-p.test \ - color-p.test color2-p.test comment9-p.test dejagnu-p.test \ - exeext4-p.test maken3-p.test maken4-p.test \ - posixsubst-tests-p.test + check-p.test check11-p.test check12-p.test check2-p.test \ + check3-p.test check4-p.test check5-p.test check6-p.test \ + check7-p.test check8-p.test check9-p.test color-p.test \ + color2-p.test comment9-p.test dejagnu-p.test exeext4-p.test \ + maken3-p.test maken4-p.test posixsubst-tests-p.test instspc_tests = instspc-squote-build.test instspc-squote-install.test \ instspc-dquote-build.test instspc-dquote-install.test \ instspc-bquote-build.test instspc-bquote-install.test \ @@ -993,6 +980,10 @@ parallel-test-driver-install.test \ parallel-tests-no-spurious-summary.test \ parallel-tests-exit-statuses.test \ parallel-tests-console-output.test \ +testsuite-summary-color.test \ +testsuite-summary-count.test \ +testsuite-summary-count-many.test \ +testsuite-summary-reference-log.test \ test-driver-end-test-results.test \ test-driver-custom-no-extra-driver.test \ test-driver-custom.test \ @@ -1397,7 +1388,7 @@ cscope cscopelist: $(TEST_SUITE_LOG): $(TEST_LOGS) - @$(am__sh_e_setup); \ + @$(am__sh_e_setup); $(am__tty_colors); \ rst_magic=":test-result:"; \ nlinit=`echo 'nl="'; echo '"'`; eval "$$nlinit"; unset nlinit; \ list='$(TEST_LOGS)'; \ @@ -1416,7 +1407,7 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) exec 0<&5; \ done; \ exec 5<&-; \ - results=`echo "$$results1" && echo "$$results2"` \ + results=`echo "$$results1" && echo "$$results2"`; \ all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \ fail=`echo "$$results" | grep -c '^FAIL'`; \ pass=`echo "$$results" | grep -c '^PASS'`; \ @@ -1424,44 +1415,44 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) xfail=`echo "$$results" | grep -c '^XFAIL'`; \ xpass=`echo "$$results" | grep -c '^XPASS'`; \ error=`echo "$$results" | grep -c '^ERROR'`; \ - fail=`expr $$fail + $$error`; \ - failures=`expr $$fail + $$xpass`; \ - all=`expr $$all - $$skip`; \ - if test "$$all" -eq 1; then tests=test; All=; \ - else tests=tests; All="All "; fi; \ - case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \ - fail=0:xpass=0:xfail=0) \ - msg="$$All$$all $$tests passed. "; \ - exit=true;; \ - fail=0:xpass=0:xfail=*) \ - msg="$$All$$all $$tests behaved as expected"; \ - if test "$$xfail" -eq 1; then xfailures=failure; \ - else xfailures=failures; fi; \ - msg="$$msg ($$xfail expected $$xfailures). "; \ - exit=true;; \ - fail=*:xpass=0:xfail=*) \ - msg="$$fail of $$all $$tests failed. "; \ - exit=false;; \ - fail=*:xpass=*:xfail=*) \ - msg="$$failures of $$all $$tests did not behave as expected"; \ - if test "$$xpass" -eq 1; then xpasses=pass; \ - else xpasses=passes; fi; \ - msg="$$msg ($$xpass unexpected $$xpasses). "; \ - exit=false;; \ - *) \ - echo >&2 "incorrect case"; exit 4;; \ - esac; \ - if test "$$skip" -ne 0; then \ - if test "$$skip" -eq 1; then \ - msg="$$msg($$skip test was not run). "; \ - else \ - msg="$$msg($$skip tests were not run). "; \ - fi; \ - fi; \ + if test `expr $$fail + $$xpass + $$error` -eq 0; then \ + success=true; \ + else \ + success=false; \ + fi; \ + br='==================='; br=$$br$$br$$br$$br; \ + result_count () \ + { \ + if test x"$$1" = x"--maybe-color"; then \ + maybe_colorize=yes; \ + elif test x"$$1" = x"--no-color"; then \ + maybe_colorize=no; \ + else \ + echo "$@: invalid 'result_count' usage" >&2; exit 4; \ + fi; \ + shift; \ + desc=$$1 count=$$2; \ + if test $$maybe_colorize = yes && test $$count -gt 0; then \ + color_start=$$3 color_end=$$std; \ + else \ + color_start= color_end=; \ + fi; \ + echo "$${color_start}# $$desc $$count$${color_end}"; \ + }; \ + create_testsuite_report () \ + { \ + result_count $$1 "TOTAL:" $$all "$$brg"; \ + result_count $$1 "PASS: " $$pass "$$grn"; \ + result_count $$1 "SKIP: " $$skip "$$blu"; \ + result_count $$1 "XFAIL:" $$xfail "$$lgn"; \ + result_count $$1 "FAIL: " $$fail "$$red"; \ + result_count $$1 "XPASS:" $$xpass "$$red"; \ + result_count $$1 "ERROR:" $$error "$$mgn"; \ + }; \ { \ echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \ $(am__rst_title); \ - echo "$$msg"; \ + create_testsuite_report --no-color; \ echo; \ echo ".. contents:: :depth: 2"; \ echo; \ @@ -1474,21 +1465,25 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) done; \ } >$(TEST_SUITE_LOG).tmp; \ mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ - if test "$$failures" -ne 0; then \ - msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \ - if test -n "$(PACKAGE_BUGREPORT)"; then \ - msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \ - fi; \ - fi; \ - test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \ - $(am__tty_colors); \ - if $$exit; then \ + if $$success; then \ col="$$grn"; \ else \ col="$$red"; \ + test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \ + fi; \ + echo "$${col}$$br$${std}"; \ + echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \ + echo "$${col}$$br$${std}"; \ + create_testsuite_report --maybe-color; \ + echo "$$col$$br$$std"; \ + if $$success; then :; else \ + echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \ + if test -n "$(PACKAGE_BUGREPORT)"; then \ + echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \ + fi; \ + echo "$$col$$br$$std"; \ fi; \ - echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \ - $$exit + $$success # Run all the tests. check-TESTS: @@ -1695,7 +1690,6 @@ check-exported-srcdir-p.log: check-exported-srcdir.test check-tests-in-builddir-p.log: check-tests-in-builddir.test check-tests_environment-p.log: check-tests_environment.test check-p.log: check.test -check10-p.log: check10.test check11-p.log: check11.test check12-p.log: check12.test check2-p.log: check2.test @@ -1775,6 +1769,12 @@ test-driver-custom-multitest-recheck.log: trivial-test-driver test-driver-custom-multitest-recheck2.log: trivial-test-driver test-driver-custom-html.log: trivial-test-driver +testsuite-summary-color.log testsuite-summary-count.log: \ + testsuite-summary-checks.sh extract-testsuite-summary + +testsuite-summary-count-many.log: trivial-test-driver +testsuite-summary-count-many.log: extract-testsuite-summary + # Dependencies valid for each test case. $(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION) diff --git a/tests/check10.test b/tests/check10.test index ef799691d..a13c85902 100755 --- a/tests/check10.test +++ b/tests/check10.test @@ -16,6 +16,8 @@ # Check singular and plural in test summaries. +# This test only makes sense for the older serial testsuite driver. +parallel_tests=no . ./defs || Exit 1 cat >> configure.in << 'END' diff --git a/tests/color.test b/tests/color.test index 2524324e7..2b56a3291 100755 --- a/tests/color.test +++ b/tests/color.test @@ -104,19 +104,23 @@ test_no_color () # Keep this in sync with lib/am/check.am:$(am__color_tests). if $FGREP '= Xalways || test -t 1 ' stdout; then # Extra verbose make, resort to laxer checks. + # Note that we also want to check that the testsuite summary is + # not unduly colorized. ( set +e # In case some grepped regex below isn't matched. # Not a useless use of cat; see above comments about grep. + cat stdout | grep "TOTAL.*:" cat stdout | grep "PASS.*:" cat stdout | grep "FAIL.*:" cat stdout | grep "SKIP.*:" cat stdout | grep "XFAIL.*:" cat stdout | grep "XPASS.*:" cat stdout | grep "ERROR.*:" - # To check that the testsuite summary is not unduly colorized. - cat stdout | grep '====' cat stdout | grep 'test.*expected' cat stdout | grep 'test.*not run' + cat stdout | grep '====' + cat stdout | grep '[Ss]ee .*test-suite\.log' + cat stdout | grep '[Tt]estsuite summary' ) | grep "$esc" && Exit 1 : For shells with broken 'set -e' else diff --git a/tests/color2.test b/tests/color2.test index 758ccd468..21d67ac21 100755 --- a/tests/color2.test +++ b/tests/color2.test @@ -117,19 +117,23 @@ test_no_color () # Keep this in sync with lib/am/check.am:$(am__color_tests). if $FGREP '= Xalways || test -t 1 ' stdout; then # Extra verbose make, resort to laxer checks. + # Note that we also want to check that the testsuite summary is + # not unduly colorized. ( set +e # In case some grepped regex below isn't matched. # Not a useless use of cat; see above comments about grep. + cat stdout | grep "TOTAL.*:" cat stdout | grep "PASS.*:" cat stdout | grep "FAIL.*:" cat stdout | grep "SKIP.*:" cat stdout | grep "XFAIL.*:" cat stdout | grep "XPASS.*:" cat stdout | grep "ERROR.*:" - # To check that the testsuite summary is not unduly colorized. - cat stdout | grep '====' cat stdout | grep 'test.*expected' cat stdout | grep 'test.*not run' + cat stdout | grep '====' + cat stdout | grep '[Ss]ee .*test-suite\.log' + cat stdout | grep '[Tt]estsuite summary' ) | grep "$esc" && Exit 1 : For shells with broken 'set -e' else diff --git a/tests/extract-testsuite-summary b/tests/extract-testsuite-summary new file mode 100644 index 000000000..a3a0d87f3 --- /dev/null +++ b/tests/extract-testsuite-summary @@ -0,0 +1,15 @@ +#! /usr/bin/env perl +# Extract the testsuite summary generated by the parallel-tests harness +# from the output of "make check". + +use warnings FATAL => 'all'; +use strict; + +my $br = '=' x 76; +my @sections = (''); +while (<>) + { + push @sections, $_, '' if /$br/; + $sections[-1] .= $_ if !/$br/; + } +print @sections[1..$#sections-1]; diff --git a/tests/parallel-tests-empty-testlogs.test b/tests/parallel-tests-empty-testlogs.test index 593dce315..e3d77ea21 100755 --- a/tests/parallel-tests-empty-testlogs.test +++ b/tests/parallel-tests-empty-testlogs.test @@ -55,7 +55,7 @@ $AUTOMAKE -a no_test_has_run () { ls -1 *.log | grep -v '^test-suite\.log$' | grep . && Exit 1 - grep ' 0 tests passed' test-suite.log + grep '^# TOTAL: *0$' test-suite.log : } diff --git a/tests/parallel-tests-log-override-recheck.test b/tests/parallel-tests-log-override-recheck.test index 8a50d293f..613bea1da 100755 --- a/tests/parallel-tests-log-override-recheck.test +++ b/tests/parallel-tests-log-override-recheck.test @@ -65,7 +65,9 @@ TEST_SUITE_LOG=my.log $MAKE -e recheck >stdout \ && { cat stdout; Exit 1; } cat stdout ls -l -grep '^2 of 2 .*failed' stdout +grep '^# TOTAL: *2$' stdout +grep '^# FAIL: *1$' stdout +grep '^# ERROR: *1$' stdout for x in stdout my.log; do $FGREP foo.test $x && Exit 1 $FGREP bar.test $x @@ -77,7 +79,9 @@ BAZ_EXIT_STATUS=0 TEST_SUITE_LOG=my2.log $MAKE -e recheck >stdout \ && { cat stdout; Exit 1; } cat stdout ls -l -grep '^1 of 2 .*failed' stdout +grep '^# TOTAL: *2$' stdout +grep '^# PASS: *1$' stdout +grep '^# ERROR: *1$' stdout $FGREP foo.test stdout && Exit 1 $FGREP bar.test stdout $FGREP baz.test stdout diff --git a/tests/parallel-tests-no-color-in-log.test b/tests/parallel-tests-no-color-in-log.test new file mode 100755 index 000000000..ab3f371b3 --- /dev/null +++ b/tests/parallel-tests-no-color-in-log.test @@ -0,0 +1,62 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Colorized output from the testsuite report shouldn't end up in log files. + +parallel_tests=yes +. ./defs || Exit 1 + +esc='' + +# Check that grep can parse nonprinting characters. +# BSD 'grep' works from a pipe, but not a seekable file. +# GNU or BSD 'grep -a' works on files, but is not portable. +case `echo "$esc" | $FGREP "$esc"` in + "$esc") ;; + *) echo "$me: fgrep can't parse nonprinting characters" >&2; Exit 77;; +esac + +TERM=ansi; export TERM + +cat >>configure.in <Makefile.am <<'END' +LOG_COMPILER = $(SHELL) +AUTOMAKE_OPTIONS = color-tests parallel-tests +TESTS = pass fail skip xpass xfail error +XFAIL_TESTS = xpass xfail +END + +# FIXME: creative quoting to please maintainer-check. +echo exit '0' > pass +echo exit '0' > xpass +echo exit '1' > fail +echo exit '1' > xfail +echo exit '77' > skip +echo exit '99' > error + +$ACLOCAL +$AUTOCONF +$AUTOMAKE --add-missing + +./configure +mv config.log config-log # Avoid possible false positives below. +AM_COLOR_TESTS=always $MAKE -e check && Exit 1 +$FGREP "$esc" *.log && Exit 1 + +: diff --git a/tests/parallel-tests-no-spurious-summary.test b/tests/parallel-tests-no-spurious-summary.test index 8b9547a6f..7fb95f56a 100755 --- a/tests/parallel-tests-no-spurious-summary.test +++ b/tests/parallel-tests-no-spurious-summary.test @@ -62,7 +62,10 @@ grep '^:test-result: SKIP$' foo.log grep '^:test-result:FAIL$' bar.log grep '^:test-result: XPASS$' bar.log -$EGREP '^(All)? 2 tests? passed' output -$EGREP 'did not behave|unexpected pass|not run' output && Exit 1 +grep '^# TOTAL: *2$' output +grep '^# PASS: *2$' output +for result in FAIL XFAIL XPASS SKIP ERROR; do + grep "^# $result: *0$" output +done : diff --git a/tests/parallel-tests-unreadable-log.test b/tests/parallel-tests-unreadable-log.test index 572f03df5..e8e101b9b 100755 --- a/tests/parallel-tests-unreadable-log.test +++ b/tests/parallel-tests-unreadable-log.test @@ -54,7 +54,7 @@ chmod a-r foo.log bar.log test ! -r foo.log || Exit 77 $MAKE test-suite.log >stdout && { cat stdout; Exit 1; } cat stdout -grep '^2 of 2 tests failed *$' stdout -grep '^2 of 2 tests failed\. *$' test-suite.log +grep '^# ERROR: *2$' stdout +grep '^# ERROR: *2$' test-suite.log : diff --git a/tests/parallel-tests.test b/tests/parallel-tests.test index 95f78e736..44e8846cb 100755 --- a/tests/parallel-tests.test +++ b/tests/parallel-tests.test @@ -78,8 +78,6 @@ test `grep -c '^ERROR:' mylog.log` -eq 1 test -f baz.log test -f bar.log test -f foo.log -# The summary should be formatted correctly. -grep 'failedn' stdout && Exit 1 # clean should remove all log files (but not more). : > unrelated.log @@ -113,7 +111,9 @@ test -f foo.log grep foo.test stdout grep bar.test stdout && Exit 1 grep baz.test stdout && Exit 1 -grep '2.*tests.*failed' stdout +grep '^# PASS: *1$' stdout +grep '^# FAIL: *1$' stdout +grep '^# ERROR: *1$' stdout # Now, explicitly retry with all test logs already updated, and ensure # that the summary is still displayed. @@ -122,7 +122,9 @@ cat stdout grep foo.test stdout && Exit 1 grep bar.test stdout && Exit 1 grep baz.test stdout && Exit 1 -grep '2.*tests.*failed' stdout +grep '^# PASS: *1$' stdout +grep '^# FAIL: *1$' stdout +grep '^# ERROR: *1$' stdout # Lazily rerunning only foo should only rerun this one test. env RECHECK_LOGS=foo.log $MAKE -e check > stdout && { cat stdout; Exit 1; } @@ -130,7 +132,9 @@ cat stdout grep foo.test stdout grep bar.test stdout && Exit 1 grep baz.test stdout && Exit 1 -grep '2.*tests.*failed' stdout +grep '^# PASS: *1$' stdout +grep '^# FAIL: *1$' stdout +grep '^# ERROR: *1$' stdout # Test VERBOSE. env VERBOSE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; } diff --git a/tests/parallel-tests3.test b/tests/parallel-tests3.test index 9f0ce4289..8153eebe2 100755 --- a/tests/parallel-tests3.test +++ b/tests/parallel-tests3.test @@ -70,7 +70,7 @@ $MAKE -j4 check >> stdout cd .. kill $! cat parallel/stdout -test `grep -c PASS parallel/stdout` -eq 8 +test `grep -c '^PASS:' parallel/stdout` -eq 8 # Wait long enough so that there are no open files any more # when the post-test cleanup runs. diff --git a/tests/parallel-tests6.test b/tests/parallel-tests6.test index 599225026..e0c535edd 100755 --- a/tests/parallel-tests6.test +++ b/tests/parallel-tests6.test @@ -36,6 +36,8 @@ $AUTOMAKE -a ./configure $MAKE check >stdout || { cat stdout; Exit 1; } cat stdout -grep '0 tests' stdout +for x in TOTAL PASS FAIL XPASS FAIL SKIP ERROR; do + grep "^# $x: *0$" stdout +done Exit 0 diff --git a/tests/parallel-tests9.test b/tests/parallel-tests9.test index e6191d922..2a6922e39 100755 --- a/tests/parallel-tests9.test +++ b/tests/parallel-tests9.test @@ -58,13 +58,18 @@ $AUTOMAKE -a ./configure $MAKE check >stdout && { cat stdout; Exit 1; } cat stdout +grep '^# PASS: *1$' stdout +grep '^# FAIL: *1$' stdout +grep '^# ERROR: *1$' stdout $MAKE recheck >stdout && { cat stdout; Exit 1; } cat stdout grep foo.test stdout && Exit 1 grep bar.test stdout || Exit 1 grep baz.test stdout || Exit 1 -grep '2 of 2.*failed' stdout +grep '^# PASS: *0$' stdout +grep '^# FAIL: *1$' stdout +grep '^# ERROR: *1$' stdout # If we cannot read the log file, then redo it as well. chmod a-r foo.log diff --git a/tests/test-driver-custom-multitest.test b/tests/test-driver-custom-multitest.test index 2366b5b83..0316f0ff1 100755 --- a/tests/test-driver-custom-multitest.test +++ b/tests/test-driver-custom-multitest.test @@ -43,6 +43,7 @@ TESTS = \ pass-xpass-fail-xfail-skip-error.t END +expected_tests=23 expected_pass=10 expected_fail=5 expected_skip=4 @@ -160,11 +161,14 @@ for vpath in : false; do grep "^SKIP: $tst\.t, testcase 5" stdout grep "^ERROR: $tst\.t, testcase 6" stdout - # Check testsuite summary printed on console. - sed -e 's/[()]/ /g' -e 's/^/ /' stdout > t - grep ' 7 of 19 ' t - grep ' 1 unexpected pass' t - grep ' 4 test.* not run' t + # Check counts of testcases in testsuite summary printed on console. + grep "^# TOTAL: *$expected_tests$" stdout + grep "^# PASS: *$expected_pass$" stdout + grep "^# XPASS: *$expected_xpass$" stdout + grep "^# FAIL: *$expected_fail$" stdout + grep "^# XFAIL: *$expected_xfail$" stdout + grep "^# SKIP: *$expected_skip$" stdout + grep "^# ERROR: *$expected_error$" stdout # Check that the content of, and only of, the test logs with at least # one failing test case has been copied into `test-suite.log'. Note @@ -188,6 +192,14 @@ for vpath in : false; do test `grep -c '^SKIP:' stdout` -eq 2 test `grep -c '^XFAIL:' stdout` -eq 1 $EGREP '^(FAIL|XPASS|ERROR)' stdout && Exit 1 + # Check counts of testcases in testsuite summary printed on console. + grep "^# TOTAL: *7$" stdout + grep "^# PASS: *4$" stdout + grep "^# XPASS: *0$" stdout + grep "^# FAIL: *0$" stdout + grep "^# XFAIL: *1$" stdout + grep "^# SKIP: *2$" stdout + grep "^# ERROR: *0$" stdout cd $srcdir diff --git a/tests/test-driver-end-test-results.test b/tests/test-driver-end-test-results.test index 03af51b7f..700870086 100755 --- a/tests/test-driver-end-test-results.test +++ b/tests/test-driver-end-test-results.test @@ -85,8 +85,11 @@ $MAKE check >output 2>&1 || { cat output; Exit 1; } cat output cat a.log; cat b.log; cat c.log; cat d.log # For debugging. -$EGREP '^(All)? 3 tests? passed' output -grep '^(1 test.* not run' output -$EGREP 'did not behave|unexpected pass' output && Exit 1 +grep '^# TOTAL: *4$' output +grep '^# PASS: *3$' output +grep '^# SKIP: *1$' output +for result in XPASS FAIL XFAIL ERROR; do + grep "^# $result: *0$" output +done : diff --git a/tests/testsuite-summary-checks.sh b/tests/testsuite-summary-checks.sh new file mode 100755 index 000000000..a4de86499 --- /dev/null +++ b/tests/testsuite-summary-checks.sh @@ -0,0 +1,106 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check the testsuite summary with the parallel-tests harness. This +# script is meant to be sourced by other test script, so that it can +# be used to check different scenarios (colorized and non-colorized +# testsuite output, packages with and without bug-report addresses, +# testsuites in subdirectories, ...) + +parallel_tests=yes +. ./defs || Exit 1 + +case $use_colors in + yes) + AM_COLOR_TESTS=always; export AM_COLOR_TESTS + TERM=ansi; export TERM + am_opts='parallel-tests color-tests' + ;; + no) + am_opts='parallel-tests' + ;; + *) + fatal_ "invalid use_colors='$use_colors'";; +esac + +cat > configure.in < Makefile.am <<'END' +TEST_EXTENSIONS = .t +T_LOG_COMPILER = $(SHELL) $(srcdir)/log-compiler +## Will be defined at runtime. +TESTS = +$(TESTS): +END + +cat > log-compiler <<'END' +#!/bin/sh +case "$1" in + pass*|*/pass*|xpass*|*/xpass*) exit 0;; + fail*|*/fail*|xfail*|*/xfail*) exit 1;; + skip*|*/skip*) exit 77;; + error*|/error*) exit 99;; + copy*|*/copy*) cat "$1";; + *) exit 99;; +esac +END + +# Quite complexish, but allow the tests in client scripts to be written +# in a "data-driven fashion". +do_check () +{ + cat > summary.exp + expect_failure=false + xfail_tests='' + tests="TESTS='$*'" + for t in $*; do + case $t in fail*|xpass*|error*) expect_failure=:;; esac + case $t in xfail*|xpass*) xfail_tests="$xfail_tests $t";; esac + done + test -z "$xfail_tests" || xfail_tests="XFAIL_TESTS='$xfail_tests'" + st=0 + eval "env $tests $xfail_tests \$MAKE -e check > stdout || st=\$?" + cat stdout + if $expect_failure; then + test $st -gt 0 || Exit 1 + else + test $st -eq 0 || Exit 1 + fi + $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \ + || fatal_ "cannot extract testsuite summary" + cat summary.exp + cat summary.got + if test $use_colors = yes; then + # Use cmp, not diff, because the files might contain binary data. + compare=cmp + else + compare=diff + fi + $compare summary.exp summary.got || Exit 1 +} + +br='============================================================================' + +$ACLOCAL +$AUTOCONF +$AUTOMAKE --add-missing + +: diff --git a/tests/testsuite-summary-color.test b/tests/testsuite-summary-color.test new file mode 100755 index 000000000..94937445a --- /dev/null +++ b/tests/testsuite-summary-color.test @@ -0,0 +1,199 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check coloring of the testsuite summary. + +. ./defs-static || Exit 1 + +use_colors=yes +use_vpath=no + +. "$testsrcdir"/testsuite-summary-checks.sh || Exit 99 + +./configure + +# ANSI colors. +red='' +grn='' +lgn='' +blu='' +mgn='' +brg=''; +std=''; + +success_header="\ +${grn}${br}${std} +${grn}Testsuite summary for GNU AutoFoo 7.1${std} +${grn}${br}${std}" + +success_footer=${grn}${br}${std} + +failure_header="\ +${red}${br}${std} +${red}Testsuite summary for GNU AutoFoo 7.1${std} +${red}${br}${std}" + +failure_footer="\ +${red}${br}${std} +${red}See ./test-suite.log${std} +${red}Please report to bug-automake@gnu.org${std} +${red}${br}${std}" + +do_check '' <. + +# Check test counts in the testsuite summary, with test drivers allowing +# multiple test results per test script, and for a huge number of tests. +# Incidentally, this test also checks that the testsuite summary doesn't +# give any bug-report address if it's not defined. + +parallel_tests=yes +. ./defs || Exit 1 + +for s in trivial-test-driver extract-testsuite-summary; do + cp "$testsrcdir/$s" . || fatal_ "failed to fetch auxiliary script $s" +done + +br='============================================================================' + +header="\ +${br} +Testsuite summary for $me 1.0 +${br}" + +footer="\ +${br} +See ./test-suite.log +${br}" + +echo AC_OUTPUT >> configure.in + +cat > Makefile.am << 'END' +TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver +TESTS = all.test +# Without this, the test driver will be horrendously slow. +END + +cat > all.test <<'END' +#!/bin/sh +cat results.txt || { echo ERROR: weird; exit 99; } +END +chmod a+x all.test + +$PERL -w -e ' + use warnings FATAL => "all"; + use strict; + + # FIXME: we would like this to be 1000 or even 10000, but the current + # implementation is too slow to handle that :-( + my $base = 5; + my %count = ( + TOTAL => $base * 1000, + PASS => $base * 700, + SKIP => $base * 200, + XFAIL => $base * 80, + FAIL => $base * 10, + XPASS => $base * 7, + ERROR => $base * 3, + ); + my @results = qw/PASS SKIP XFAIL FAIL XPASS ERROR/; + + open (RES, ">results.txt") or die "opening results.txt: $!\n"; + open (CNT, ">count.txt") or die "opening count.txt: $!\n"; + + printf CNT "# %-6s %d\n", "TOTAL:", $count{TOTAL}; + for my $res (@results) + { + my $uc_res = uc $res; + print STDERR "Generating list of $res ...\n"; + for (1..$count{$res}) + { + print RES "$uc_res: $_\n"; + } + printf CNT "# %-6s %d\n", $res . ":", $count{$res}; + } +' + +(echo "$header" && cat count.txt && echo "$footer") > summary.exp + +$ACLOCAL +$AUTOMAKE -a +$AUTOCONF + +./configure + +($MAKE check || : > make.fail) | tee stdout +test -f make.fail + +$PERL -w extract-testsuite-summary stdout > summary.got +cat summary.exp +cat summary.got +diff summary.exp summary.got || Exit 1 + +: diff --git a/tests/testsuite-summary-count.test b/tests/testsuite-summary-count.test new file mode 100755 index 000000000..3e24e3a31 --- /dev/null +++ b/tests/testsuite-summary-count.test @@ -0,0 +1,176 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check test counts in the testsuite summary. + +. ./defs-static || Exit 1 + +use_colors=no +use_vpath=no + +. "$testsrcdir"/testsuite-summary-checks.sh || Exit 99 + +seq_ () +{ + case $# in + 2) l=$1 u=$2;; + *) fatal_ "incorrect usage of 'seq_' function";; + esac + seq $1 $2 || { + i=$l + while test $i -le $u; do + echo $i + i=`expr $i + 1` + done + } +} + +./configure + +header="\ +${br} +Testsuite summary for GNU AutoFoo 7.1 +${br}" + +success_footer=${br} + +failure_footer="\ +${br} +See ./test-suite.log +Please report to bug-automake@gnu.org +${br}" + +# Corner cases. + +do_check '' <. + +# Check that the global testsuite log file referenced in the testsuite +# summary and in the global testsuite log itself is correct. + +parallel_tests=yes +. ./defs || Exit 1 + +mv configure.in configure.stub + +cat > fail << 'END' +#!/bin/sh +exit 1 +END +chmod a+x fail + +cat configure.stub - > configure.in <<'END' +AC_OUTPUT +END + +cat > Makefile.am << 'END' +TEST_SUITE_LOG = my_test_suite.log +TESTS = fail +END + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +mkdir build +cd build + +../configure + +$MAKE check >stdout && { cat stdout; Exit 1; } +cat stdout +grep '^See \./my_test_suite\.log$' stdout + +mkdir bar +TEST_SUITE_LOG=bar/bar.log $MAKE -e check >stdout && { cat stdout; Exit 1; } +cat stdout +grep '^See \./bar/bar\.log$' stdout + +cd .. + +echo SUBDIRS = sub > Makefile.am +mkdir sub +echo TESTS = fail > sub/Makefile.am +mv fail sub + +cat configure.stub - > configure.in <<'END' +AC_CONFIG_FILES([sub/Makefile]) +AC_OUTPUT +END + +$ACLOCAL --force +$AUTOCONF --force +$AUTOMAKE + +./configure +$MAKE check >stdout && { cat stdout; Exit 1; } +cat stdout +grep '^See sub/test-suite\.log$' stdout +cd sub +$MAKE check >stdout && { cat stdout; Exit 1; } +cat stdout +grep '^See sub/test-suite\.log$' stdout +cd .. + +TEST_SUITE_LOG=foo.log $MAKE -e check >stdout && { cat stdout; Exit 1; } +cat stdout +grep '^See sub/foo\.log$' stdout + +: diff --git a/tests/trivial-test-driver b/tests/trivial-test-driver index 113e158fc..4b4350688 100644 --- a/tests/trivial-test-driver +++ b/tests/trivial-test-driver @@ -54,30 +54,37 @@ done ## Run the test script, get test cases results, display them on console. -tmp_out=$log_file-out.tmp -tmp_res=$log_file-res.tmp +tmp_output=$log_file-output.tmp +tmp_results=$log_file-results.tmp +tmp_status=$log_file-status.tmp -"$@" 2>&1 | tee $tmp_out | ( +"$@" 2>&1 | tee $tmp_output | ( i=0 st=0 - : > $tmp_res + exec 5> $tmp_results + : > $tmp_status while read line; do + result= case $line in - PASS:*|FAIL:*|XPASS:*|XFAIL:*|SKIP:*|ERROR:*) - i=`expr $i + 1` - result=`LC_ALL=C expr "$line" : '\([A-Z]*\):.*'` - case $result in FAIL|XPASS|ERROR) st=1;; esac - # Output testcase result to console. - echo "$result: $test_name, testcase $i" - # Register testcase outcome for the log file. - echo ":test-result: $line" >> $tmp_res - echo >> $tmp_res - ;; + PASS:*) result=PASS ;; + FAIL:*) result=FAIL ;; + XPASS:*) result=XPASS ;; + XFAIL:*) result=XFAIL ;; + SKIP:*) result=SKIP ;; + ERROR:*) result=ERROR ;; esac + if test -n "$result"; then + case $result in FAIL|XPASS|ERROR) st=1;; esac + # Output testcase result to console. + echo "$result: $test_name" + # Register testcase outcome for the log file. + echo ":test-result: $line" >&5 + echo >&5 + fi done - exit $st -) + test $st -eq 0 || echo fail > $tmp_status +) | awk '{ print $0 ", testcase " NR }' -if test $? -eq 0; then +if test ! -s $tmp_status; then global_result=PASS else global_result=FAIL @@ -89,13 +96,13 @@ fi echo "$global_result: $test_name" echo "$global_result: $test_name" | sed 's/./=/g' echo - cat $tmp_res + cat $tmp_results echo echo -------------------- echo - cat $tmp_out + cat $tmp_output } > $log_file -rm -f $tmp_out $tmp_res +rm -f $tmp_output $tmp_results $tmp_status ## And we're done. -- 2.47.2