From 8f33eec4110c89f2963a4e3910c8149c59e63b23 Mon Sep 17 00:00:00 2001 From: Ralf Wildenhues Date: Sat, 28 Mar 2009 22:58:34 +0100 Subject: [PATCH] parallel-tests: redo lazy checking: recheck and RECHECK_LOGS. Replace the LAZY_TEST_SUITE API with a simpler yet more powerful one: RECHECK_LOGS specifies those tests which are to be removed in any case before testing. Provide a `recheck' convenience target to set RECHECK_LOGS to all failed and unexpectedly passed tests. Document several ways to limit the set of tests run. * lib/am/check.am [PARALLEL_TESTS] (RECHECK_LOGS): New variable, default to $(TESTS_LOGS). (check-TESTS): Remove $(RECHECK_LOGS) not $(TEST_LOGS). Drop use of LAZY_TEST_SUITE. ($(TEST_SUITE_LOG)): Do not output note about lazy rerun, as LAZY_TEST_SUITE is gone. (recheck): New target. (recheck-am, recheck-TESTS): New internal targets. * doc/automake.texi (Tests): Update @vindex for TESTS and TEST_LOGS. Replace description of LAZY_TEST_SUITE with a list of ways the set of tests to be run can be modified. Document RECHECK_LOGS and the recheck target. * tests/defs.in: Unset RECHECK_LOGS not LAZY_TEST_SUITE. * tests/parallel-tests.test: Adjust, replacing LAZY_TEST_SUITE with corresponding RECHECK_LOGS settings, and add another RECHECK_LOGS test. * tests/parallel-tests9.test: New test, test `recheck'. * tests/Makefile.am: Update. Suggestion and different implementation by Akim Demaille. Signed-off-by: Ralf Wildenhues --- ChangeLog | 26 +++++++++++++ doc/automake.texi | 55 ++++++++++++++++++++++---- lib/Automake/tests/Makefile.in | 25 +++++++----- lib/am/check.am | 29 +++++++++----- tests/Makefile.am | 1 + tests/Makefile.in | 26 ++++++++----- tests/defs.in | 2 +- tests/parallel-tests.test | 15 ++++++-- tests/parallel-tests9.test | 70 ++++++++++++++++++++++++++++++++++ 9 files changed, 210 insertions(+), 39 deletions(-) create mode 100755 tests/parallel-tests9.test diff --git a/ChangeLog b/ChangeLog index a33abf9b5..57ca6d82f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,30 @@ 2009-03-28 Ralf Wildenhues + Akim Demaille + + parallel-tests: redo lazy checking: recheck and RECHECK_LOGS. + Replace the LAZY_TEST_SUITE API with a simpler yet more powerful + one: RECHECK_LOGS specifies those tests which are to be removed + in any case before testing. Provide a `recheck' convenience + target to set RECHECK_LOGS to all failed and unexpectedly passed + tests. Document several ways to limit the set of tests run. + * lib/am/check.am [PARALLEL_TESTS] (RECHECK_LOGS): New variable, + default to $(TESTS_LOGS). + (check-TESTS): Remove $(RECHECK_LOGS) not $(TEST_LOGS). Drop use + of LAZY_TEST_SUITE. + ($(TEST_SUITE_LOG)): Do not output note about lazy rerun, as + LAZY_TEST_SUITE is gone. + (recheck): New target. + (recheck-am, recheck-TESTS): New internal targets. + * doc/automake.texi (Tests): Update @vindex for TESTS and + TEST_LOGS. Replace description of LAZY_TEST_SUITE with a list + of ways the set of tests to be run can be modified. Document + RECHECK_LOGS and the recheck target. + * tests/defs.in: Unset RECHECK_LOGS not LAZY_TEST_SUITE. + * tests/parallel-tests.test: Adjust, replacing LAZY_TEST_SUITE + with corresponding RECHECK_LOGS settings. + * tests/parallel-tests9.test: New tests. + * tests/Makefile.am: Update. + Suggestion and different implementation by Akim Demaille. parallel-tests: do not mark check-TESTS as `.MAKE'. * lib/am/check.am [PARALLEL_TESTS] (.MAKE): Remove check-TESTS. diff --git a/doc/automake.texi b/doc/automake.texi index bf41acb40..84a8a216e 100644 --- a/doc/automake.texi +++ b/doc/automake.texi @@ -8388,7 +8388,7 @@ This test driver is still experimental and may undergo changes in order to satisfy additional portability requirements. @vindex TEST_SUITE_LOG -@vindex TEST_LOGS +@vindex TESTS The driver operates by defining a set of @command{make} rules to create a summary log file, @code{TEST_SUITE_LOG}, which defaults to @file{test-suite.log} and requires a @file{.log} suffix. This file @@ -8397,6 +8397,7 @@ depends upon log files created for each single test program listed in corresponding tests. @vindex TEST_EXTENSIONS +@vindex TEST_LOGS Each log file is created when the corresponding test has completed. The set of log files is listed in the read-only variable @code{TEST_LOGS}, and defaults to @code{TESTS}, with the executable @@ -8468,12 +8469,52 @@ Tests can exit with an exit status of 99 to signal such a @emph{hard error}. Unless the variable @code{DISABLE_HARD_ERRORS} is set to a nonempty value, such tests will be counted as failed. -@vindex LAZY_TEST_SUITE -By default, all tests listed in @code{TESTS} are run upon @code{make -check}. When @code{LAZY_TEST_SUITE} is nonempty, then log files of -a previous run are not removed before starting the test suite, so only -tests that have not yet been completed are run, as well as tests that -have been modified after the previous run. +By default, the test suite driver will run all tests, but there are +several ways to limit the set of tests that are run: + +@itemize @bullet +@item +You can set the @code{TESTS} variable, similarly to how you can with +the simple test driver from the previous section. For example, you can +use a command like this to run only a subset of the tests: + +@example +env TESTS="foo.test bar.test" make -e check +@end example + +@item +You can set the @code{TEST_LOGS} variable. By default, this variable is +computed at @command{make} run time from the value of @code{TESTS} as +described above. For example, you can use the following: + +@example +set x subset*.log; shift +env TEST_LOGS="foo.log $*" make -e check +@end example + +@item +@vindex RECHECK_LOGS +@cindex lazy test execution +By default, the test driver removes all old per-test log files before it +starts running tests to regenerate them. The variable +@code{RECHECK_LOGS} contains the set of log files which are removed. +@code{RECHECK_LOGS} defaults to @code{TEST_LOGS}, which means all tests +need to be rechecked. By overriding this variable, you can choose which +tests need to be reconsidered. For example, you can lazily rerun only +those tests which are outdated, i.e., older than their prerequisite test +files, by setting this variable to the empty value: + +@example +env RECHECK_LOGS= make -e check +@end example + +@item +@trindex recheck +You can ensure that all tests are rerun which have failed or passed +unexpectedly, by running @code{make recheck} in the test directory. +This convenience target will set @code{RECHECK_LOGS} appropriately +before invoking the main test driver. +@end itemize In order to guarantee an ordering between tests even with @code{make -j@var{N}}, dependencies between the corresponding log files may be diff --git a/lib/Automake/tests/Makefile.in b/lib/Automake/tests/Makefile.in index 7cf3ca72f..3ff98473c 100644 --- a/lib/Automake/tests/Makefile.in +++ b/lib/Automake/tests/Makefile.in @@ -123,6 +123,7 @@ elif test -f "$$f"; then dir=; \ else dir="$(srcdir)/"; fi; \ tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM; \ $(TESTS_ENVIRONMENT) +RECHECK_LOGS = $(TEST_LOGS) TEST_SUITE_LOG = test-suite.log TEST_SUITE_HTML = $(TEST_SUITE_LOG:.log=.html) am__test_logs1 = $(TESTS:=.log) @@ -362,9 +363,6 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) done; \ } >$(TEST_SUITE_LOG).tmp; \ mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ - if test -n '$(LAZY_TEST_SUITE)'; then \ - msg="$${msg}(tests were rerun lazily). "; \ - fi; \ if test "$$failures" -ne 0; then \ msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \ if test -n "$(PACKAGE_BUGREPORT)"; then \ @@ -384,15 +382,23 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) # Run all the tests. check-TESTS: - @list='$(TEST_LOGS)'; if test -z '$(LAZY_TEST_SUITE)' \ - && test -n "$$list"; then \ - rm -f $$list; \ - fi + @list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG) @set_logs=; if test "X$(TEST_LOGS)" = X.log; then \ set_logs=TEST_LOGS=; \ fi; \ $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) $$set_logs +recheck-TESTS: + @list='$(TEST_LOGS)'; \ + logs=`for f in $$list; do \ + if read line < $$f; then \ + case $$line in FAIL*|XPASS*) echo $$f;; esac; \ + else echo $$f; fi; \ + done | tr '\012\015' ' '`; \ + $(MAKE) $(AM_MAKEFLAGS) check-TESTS RECHECK_LOGS="$$logs" + +recheck-am: recheck-TESTS +recheck: recheck-am .log.html: @list='$(RST2HTML) $$RST2HTML rst2html rst2html.py'; \ @@ -545,7 +551,7 @@ ps-am: uninstall-am: -.MAKE: check-am check-html install-am install-strip +.MAKE: check-am check-html install-am install-strip recheck-am .PHONY: all all-am check check-TESTS check-am check-html clean \ clean-generic distclean distclean-generic distdir dvi dvi-am \ @@ -556,7 +562,8 @@ uninstall-am: install-ps install-ps-am install-strip installcheck \ installcheck-am installdirs maintainer-clean \ maintainer-clean-generic mostlyclean mostlyclean-generic pdf \ - pdf-am ps ps-am uninstall uninstall-am + pdf-am ps ps-am recheck recheck-TESTS recheck-am uninstall \ + uninstall-am # Tell versions [3.59,3.63) of GNU make to not export all variables. diff --git a/lib/am/check.am b/lib/am/check.am index 1880de57b..d7ea96e17 100644 --- a/lib/am/check.am +++ b/lib/am/check.am @@ -52,7 +52,7 @@ include inst-vars.am ## It provides special support for "unit tests", that is to say, ## tests that (once run) no longer need to be re-compiled and ## re-run at each "make check", unless their sources changed. To -## enable unit-test supports, define LAZY_TEST_SUITE. In such a +## enable unit-test supports, set RECHECK_LOGS to empty. In such a ## setting, that heavily relies on correct dependencies, its users may ## prefer to define EXTRA_PROGRAMS instead of check_PROGRAMS, because ## it allows intertwined compilation and execution of the tests. @@ -202,9 +202,6 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) done; \ } >$(TEST_SUITE_LOG).tmp; \ mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ - if test -n '$(LAZY_TEST_SUITE)'; then \ - msg="$${msg}(tests were rerun lazily). "; \ - fi; \ if test "$$failures" -ne 0; then \ msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \ if test -n "$(PACKAGE_BUGREPORT)"; then \ @@ -222,13 +219,12 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \ $$exit +RECHECK_LOGS = $(TEST_LOGS) + # Run all the tests. check-TESTS: -## Expand $(TEST_LOGS) only once, to avoid exceeding line length limits. - @list='$(TEST_LOGS)'; if test -z '$(LAZY_TEST_SUITE)' \ - && test -n "$$list"; then \ - rm -f $$list; \ - fi +## Expand $(RECHECK_LOGS) only once, to avoid exceeding line length limits. + @list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list ## We always have to remove TEST_SUITE_LOG, to ensure its rule is run ## in any case even in lazy mode: otherwise, if no test needs rerunning, ## or a prior run plus reruns all happen within the same timestamp @@ -243,6 +239,21 @@ check-TESTS: fi; \ $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) $$set_logs +## Rerun all FAILed or XPASSed tests (as well as all whose logs are out +## of date or do not exist). +recheck-TESTS: + @list='$(TEST_LOGS)'; \ + logs=`for f in $$list; do \ + if read line < $$f; then \ + case $$line in FAIL*|XPASS*) echo $$f;; esac; \ + else echo $$f; fi; \ + done | tr '\012\015' ' '`; \ + $(MAKE) $(AM_MAKEFLAGS) check-TESTS RECHECK_LOGS="$$logs" + +recheck-am: recheck-TESTS +recheck: recheck-am +.PHONY: recheck recheck-am recheck-TESTS +.MAKE: recheck-am ## -------------- ## ## Produce HTML. ## diff --git a/tests/Makefile.am b/tests/Makefile.am index daecf3434..1b293cbe8 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -487,6 +487,7 @@ parallel-tests5.test \ parallel-tests6.test \ parallel-tests7.test \ parallel-tests8.test \ +parallel-tests9.test \ parse.test \ percent.test \ percent2.test \ diff --git a/tests/Makefile.in b/tests/Makefile.in index 183832add..6c481cabe 100644 --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -125,6 +125,7 @@ elif test -f "$$f"; then dir=; \ else dir="$(srcdir)/"; fi; \ tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM; \ $(TESTS_ENVIRONMENT) +RECHECK_LOGS = $(TEST_LOGS) TEST_SUITE_LOG = test-suite.log TEST_SUITE_HTML = $(TEST_SUITE_LOG:.log=.html) TEST_EXTENSIONS = .test @@ -717,6 +718,7 @@ parallel-tests5.test \ parallel-tests6.test \ parallel-tests7.test \ parallel-tests8.test \ +parallel-tests9.test \ parse.test \ percent.test \ percent2.test \ @@ -1074,9 +1076,6 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) done; \ } >$(TEST_SUITE_LOG).tmp; \ mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ - if test -n '$(LAZY_TEST_SUITE)'; then \ - msg="$${msg}(tests were rerun lazily). "; \ - fi; \ if test "$$failures" -ne 0; then \ msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \ if test -n "$(PACKAGE_BUGREPORT)"; then \ @@ -1096,15 +1095,23 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) # Run all the tests. check-TESTS: - @list='$(TEST_LOGS)'; if test -z '$(LAZY_TEST_SUITE)' \ - && test -n "$$list"; then \ - rm -f $$list; \ - fi + @list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG) @set_logs=; if test "X$(TEST_LOGS)" = X.log; then \ set_logs=TEST_LOGS=; \ fi; \ $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) $$set_logs +recheck-TESTS: + @list='$(TEST_LOGS)'; \ + logs=`for f in $$list; do \ + if read line < $$f; then \ + case $$line in FAIL*|XPASS*) echo $$f;; esac; \ + else echo $$f; fi; \ + done | tr '\012\015' ' '`; \ + $(MAKE) $(AM_MAKEFLAGS) check-TESTS RECHECK_LOGS="$$logs" + +recheck-am: recheck-TESTS +recheck: recheck-am .log.html: @list='$(RST2HTML) $$RST2HTML rst2html rst2html.py'; \ @@ -1259,7 +1266,7 @@ ps-am: uninstall-am: -.MAKE: check-am check-html install-am install-strip +.MAKE: check-am check-html install-am install-strip recheck-am .PHONY: all all-am check check-TESTS check-am check-html clean \ clean-generic clean-local distclean distclean-generic distdir \ @@ -1270,7 +1277,8 @@ uninstall-am: install-pdf-am install-ps install-ps-am install-strip \ installcheck installcheck-am installdirs maintainer-clean \ maintainer-clean-generic mostlyclean mostlyclean-generic pdf \ - pdf-am ps ps-am uninstall uninstall-am + pdf-am ps ps-am recheck recheck-TESTS recheck-am uninstall \ + uninstall-am $(srcdir)/parallel-tests.am: gen-parallel-tests Makefile.am diff --git a/tests/defs.in b/tests/defs.in index 3b8fe9681..6f2babef4 100644 --- a/tests/defs.in +++ b/tests/defs.in @@ -299,7 +299,7 @@ unset DESTDIR # need to control (and test for) in some of the tests to ensure # backward-compatible behavior. unset DISABLE_HARD_ERRORS -unset LAZY_TEST_SUITE +unset RECHECK_LOGS unset VERBOSE echo "=== Running test $0" diff --git a/tests/parallel-tests.test b/tests/parallel-tests.test index e7ebd3db5..056ab2f38 100755 --- a/tests/parallel-tests.test +++ b/tests/parallel-tests.test @@ -20,7 +20,7 @@ # - TEST_SUITE_LOG # - dependencies between tests # - DISABLE_HARD_ERRORS -# - LAZY_TEST_SUITE +# - RECHECK_LOGS . ./defs-p || Exit 1 @@ -108,24 +108,31 @@ test -f mylog.log # Note that the previous test and this one taken together expose the timing # issue that requires the check-TESTS rule to always remove TEST_SUITE_LOG # before running the tests lazily. -env LAZY_TEST_SUITE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; } +env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; } cat stdout test -f foo.log grep foo.test stdout grep bar.test stdout && Exit 1 grep baz.test stdout && Exit 1 grep '2.*tests.*failed' stdout -grep 'lazily' stdout # Now, explicitly retry with all test logs already updated, and ensure # that the summary is still displayed. -env LAZY_TEST_SUITE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; } +env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; } cat stdout grep foo.test stdout && Exit 1 grep bar.test stdout && Exit 1 grep baz.test stdout && Exit 1 grep '2.*tests.*failed' stdout +# Lazily rerunning only foo should only rerun this one test. +env RECHECK_LOGS=foo.log $MAKE -e check > stdout && { cat stdout; Exit 1; } +cat stdout +grep foo.test stdout +grep bar.test stdout && Exit 1 +grep baz.test stdout && Exit 1 +grep '2.*tests.*failed' stdout + # Test VERBOSE. env VERBOSE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; } cat stdout diff --git a/tests/parallel-tests9.test b/tests/parallel-tests9.test new file mode 100755 index 000000000..7c935d96d --- /dev/null +++ b/tests/parallel-tests9.test @@ -0,0 +1,70 @@ +#! /bin/sh +# Copyright (C) 2009 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check parallel-tests features: +# - recheck + +. ./defs-p || Exit 1 + +set -e + +cat >> configure.in << 'END' +AC_OUTPUT +END + +cat > Makefile.am << 'END' +TEST_SUITE_LOG = mylog.log +TESTS = foo.test bar.test baz.test +END + +cat >>foo.test <<'END' +#! /bin/sh +echo "this is $0" +exit 0 +END +cat >>bar.test <<'END' +#! /bin/sh +echo "this is $0" +exit 99 +END +cat >>baz.test <<'END' +#! /bin/sh +echo "this is $0" +exit 1 +END +chmod a+x foo.test bar.test baz.test + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +./configure +$MAKE check >stdout && { cat stdout; Exit 1; } +cat stdout + +$MAKE recheck >stdout && { cat stdout; Exit 1; } +cat stdout +grep foo.test stdout && Exit 1 +grep bar.test stdout || Exit 1 +grep baz.test stdout || Exit 1 + +# If we cannot read the log file, then redo it as well. +chmod a-r foo.log +$MAKE recheck >stdout && { cat stdout; Exit 1; } +cat stdout +grep foo.test stdout || Exit 1 + +: -- 2.47.2