From: Stefano Lattarini <stefano.lattarini@gmail.com>
Date: Tue, 8 May 2012 09:23:38 +0000 (+0200)
Subject: [ng] check: AM_LAZY_CHECK="yes", not RECHECK_TESTS="", for lazy re-runs
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=02f42e6e18ff489d0374c693fd1c3c45b6d78ddd;p=thirdparty%2Fautomake.git

[ng] check: AM_LAZY_CHECK="yes", not RECHECK_TESTS="", for lazy re-runs

This causes a backward-incompatibility with mainline automake, but will
also offer a cleaner API and a possibility to further simplify the
implementation of the 'check' and 'recheck' targets in the future.

* doc/automake.texi, NG-NEWS, t/README: Update.
* lib/am/check.am (am__remove_if_not_lazy_check): New internal function,
expanding to shell code that removes the given list of files unless the
'AM_LAZY_CHECK' variable is set to "yes".
(check-TESTS): Use it instead of the contents of 'RECHECK_LOGS' top
decide which .log and .trs files to remove.
(RECHECK_LOGS): Don't define anymore.
* t/parallel-tests.sh: Adjust.
* t/parallel-tests-extra-programs.sh: Likewise.
* t/test-trs-recover.sh: Likewise.
* t/tap-recheck-logs.sh: Rename ...
* t/tap-lazy-check.sh: ... to this, and adjust.
* defs: Don't unset 'RECHECK_LOGS'; unset 'AM_LAZY_CHECK' instead.
* syntax-checks.mk: Adjust some comments.
(sc_no_RECHECK_LOGS): New maintainer check, guard against uses of
the now-obsolete 'RECHECK_LOGS' variable.
(syntax_check_rules): Add it.

Signed-off-by: Stefano Lattarini <stefano.lattarini@gmail.com>
---

diff --git a/NG-NEWS b/NG-NEWS
index 1f638b6db..b28fcccd6 100644
--- a/NG-NEWS
+++ b/NG-NEWS
@@ -113,6 +113,16 @@ Parallel testsuite harness
     # "foo.test", "bar.test$(EXEEXT)" and "baz$(EXEEXT)".
     make check TESTS="foo bar baz"
 
+* The use of 'RECHECK_LOGS' is not supported anymore.  Thus, to lazily
+  re-run only the tests whose '.trs' or '.log' files are out-of-date,
+  one must now use:
+
+    make check AM_LAZY_CHECK=yes   # New valid API.
+
+  instead of:
+
+    make check RECHECK_LOGS=""     # Old API, won't work anymore.
+
 Pattern rules and suffix rules
 ==============================
 
diff --git a/defs b/defs
index 36f298ce7..32b60b634 100644
--- a/defs
+++ b/defs
@@ -162,11 +162,11 @@ unset srcdir
 unset TESTS_ENVIRONMENT AM_TESTS_ENVIRONMENT
 unset DISABLE_HARD_ERRORS
 unset AM_COLOR_TESTS
+unset AM_LAZY_CHECK
 unset TESTS
 unset XFAIL_TESTS
 unset TEST_LOGS
 unset TEST_SUITE_LOG
-unset RECHECK_LOGS
 unset VERBOSE
 for pfx in TEST_ SH_ TAP_ ''; do
   unset ${pfx}LOG_COMPILER
diff --git a/doc/automake.texi b/doc/automake.texi
index 471c2a421..046ab5f89 100644
--- a/doc/automake.texi
+++ b/doc/automake.texi
@@ -9100,30 +9100,24 @@ will write the result of the partial testsuite runs to the
 @file{partial.log}, without touching @file{test-suite.log}.
 
 @item
-@c FIXME: this should become "RECHECK_TESTS", for consistency.
-@c FIXME: this will imply a change to the API, of course.
-@vindex RECHECK_LOGS
+@vindex AM_LAZY_CHECK
 @cindex lazy test execution
-By default, the test harness removes all old per-test @file{.log} and
-@file{.trs} files before it starts running tests to regenerate them.  The
-variable @code{RECHECK_LOGS} contains the set of @file{.log} (and, by
-implication, @file{.trs}) files which are removed.  @code{RECHECK_LOGS}
-defaults to @code{TEST_LOGS}, which means all tests need to be rechecked.
-By overriding this variable, you can choose which tests need to be
-reconsidered.  For example, you can lazily rerun only those tests which
-are outdated, i.e., older than their prerequisite test files, by setting
-this variable to the empty value:
+By default, the test harness will run all the tests anew.  By setting
+the @code{AM_LAZY_CHECK} variable to @code{"yes"}, you can instruct the
+harness to re-run only those test scripts whose @file{.log} or
+@file{.trs} is out-of-date (or non-existent):
 
 @example
-make check RECHECK_LOGS=
+make check AM_LAZY_CHECK=yes
 @end example
 
+In normal circumstances, this offers a simple way to lazily rerun only
+those tests which are outdated, i.e., older than their prerequisites.
+
 @item
 @trindex recheck
 You can ensure that all tests are rerun which have failed or passed
 unexpectedly, by running @code{make recheck} in the test directory.
-This convenience target will set @code{RECHECK_LOGS} appropriately
-before invoking the main test harness.
 @end itemize
 
 @noindent
@@ -9219,8 +9213,7 @@ concurrency through the use of @command{make}'s option @option{-j};
 per-test @file{.log} and @file{.trs} files, and generation of a summary
 @file{.log} file from them;
 @item
-@code{recheck} target, @code{RECHECK_LOGS} variable, and lazy reruns
-of tests;
+@code{recheck} target and on-demand lazy reruns of tests;
 @item
 inter-test dependencies;
 @item
diff --git a/lib/am/check.am b/lib/am/check.am
index 77f31c560..428242cc4 100644
--- a/lib/am/check.am
+++ b/lib/am/check.am
@@ -326,17 +326,18 @@ $(TEST_SUITE_LOG): $(am__test_logs) $(am__test_results)
 	fi;								\
 	$$success || exit 1
 
-RECHECK_LOGS = $(TEST_LOGS)
-
 ## ------------------------------------------ ##
 ## Running all tests, or rechecking failures. ##
 ## ------------------------------------------ ##
 
+am__remove_if_not_lazy_check = \
+## Expand the given list only once, to avoid exceeding line length limits.
+  $(if $(filter yes,$(AM_LAZY_CHECK)),, \
+       list='$(strip $(1))'; test -z "$$list" || rm -f $$list)
+
 check-TESTS:
-## Here and below, we expand $(RECHECK_LOGS) only once, to avoid exceeding
-## line length limits.
-	@list='$(RECHECK_LOGS)';           test -z "$$list" || rm -f $$list
-	@list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list
+	@$(call am__remove_if_not_lazy_check,$(am__test_results))
+	@$(call am__remove_if_not_lazy_check,$(am__test_logs))
 ## We always have to remove TEST_SUITE_LOG, to ensure its rule is run
 ## in any case even in lazy mode: otherwise, if no test needs rerunning,
 ## or a prior run plus reruns all happen within the same timestamp (can
diff --git a/syntax-checks.mk b/syntax-checks.mk
index 420d73732..9804378d0 100644
--- a/syntax-checks.mk
+++ b/syntax-checks.mk
@@ -67,6 +67,7 @@ sc_no_DIST_SOURCES \
 sc_no_am_TEST_BASES \
 sc_no_am_TEST_RESULTS \
 sc_no_am_TEST_LOGS \
+sc_no_RECHECK_LOGS \
 sc_tests_no_make_e \
 sc_docs_no_make_e \
 sc_make_simple_include \
@@ -336,6 +337,21 @@ sc_no_am_TEST_BASES sc_no_am_TEST_RESULTS sc_no_am_TEST_LOGS: sc_no_am_% :
 	  exit 1; \
 	fi
 
+sc_no_RECHECK_LOGS:
+	@files="\
+	  $(xtests) \
+	  $(pms) \
+	  $(ams) \
+	  $(srcdir)/doc/*.texi \
+	  $(srcdir)/automake.in \
+	  README t/README \
+	"; \
+	if grep -F 'RECHECK_LOGS' $$files; then \
+	  echo "'RECHECK_LOGS' is obsolete and no more used." >&2; \
+	  echo "You should use 'AM_LAZY_CHECK' instead." >&2; \
+	  exit 1; \
+	fi
+
 ## "make -e" is brittle and unsafe, since it let *all* the environment
 ## win over the macro definitions in the Makefiles.  We needed it when
 ## we couldn't assume GNU make, but now that the tide has turned, it's
@@ -482,7 +498,7 @@ sc_tests_no_configure_in:
 
 ## Rule to ensure that the testsuite has been run before.  We don't depend
 ## on 'check' here, because that would be very wasteful in the common case.
-## We could run "make check RECHECK_LOGS=" and avoid toplevel races with
+## We could run "make check AM_LAZY_CHECK=yes" and avoid toplevel races with
 ## AM_RECURSIVE_TARGETS.  Suggest keeping test directories around for
 ## greppability of the Makefile.in files.
 sc_ensure_testsuite_has_run:
diff --git a/t/README b/t/README
index f1cdc5ff6..8628f1e21 100644
--- a/t/README
+++ b/t/README
@@ -27,7 +27,7 @@ Running the tests
 
   To run only tests that are newer than their last results:
 
-    make -k check RECHECK_LOGS=
+    make -k check AM_LAZY_CHECK=yes
 
   To run only selected tests:
 
diff --git a/t/parallel-tests-extra-programs.sh b/t/parallel-tests-extra-programs.sh
index a42bb5d2c..8cc2df7d2 100755
--- a/t/parallel-tests-extra-programs.sh
+++ b/t/parallel-tests-extra-programs.sh
@@ -152,7 +152,7 @@ $sleep
 echo 'int main (void) { return 0; }' > none.c
 
 st=0
-$MAKE check RECHECK_LOGS= >stdout || st=$?
+$MAKE check AM_LAZY_CHECK=yes >stdout || st=$?
 cat stdout
 ls -l
 test $st -eq 0 || Exit 1
diff --git a/t/parallel-tests.sh b/t/parallel-tests.sh
index 63949a2fb..538ee7550 100755
--- a/t/parallel-tests.sh
+++ b/t/parallel-tests.sh
@@ -20,7 +20,7 @@
 #  - make clean
 #  - dependencies between tests
 #  - TESTS redefinition at runtime (with and without test suffixes)
-#  - RECHECK_LOGS redefinition at runtime
+#  - AM_LAZY_CHECK
 
 am_parallel_tests=yes
 . ./defs || Exit 1
@@ -98,7 +98,7 @@ test -f test-suite.log
 # Note that the previous test and this one taken together expose the timing
 # issue that requires the check-TESTS rule to always remove TEST_SUITE_LOG
 # before running the tests lazily.
-$MAKE check RECHECK_LOGS= > stdout && { cat stdout; Exit 1; }
+$MAKE check AM_LAZY_CHECK=yes > stdout && { cat stdout; Exit 1; }
 cat stdout
 test -f foo.log
 grep '^PASS: foo\.test$' stdout
@@ -110,7 +110,7 @@ grep '^# ERROR: *1$' stdout
 
 # Now, explicitly retry with all test logs already updated, and ensure
 # that the summary is still displayed.
-$MAKE check RECHECK_LOGS= > stdout && { cat stdout; Exit 1; }
+$MAKE check AM_LAZY_CHECK=yes > stdout && { cat stdout; Exit 1; }
 cat stdout
 grep foo.test stdout && Exit 1
 grep bar.test stdout && Exit 1
@@ -119,16 +119,6 @@ grep '^# PASS: *1$' stdout
 grep '^# FAIL: *1$' stdout
 grep '^# ERROR: *1$' stdout
 
-# Lazily rerunning only foo should only rerun this one test.
-$MAKE check RECHECK_LOGS=foo.log > stdout && { cat stdout; Exit 1; }
-cat stdout
-grep foo.test stdout
-grep bar.test stdout && Exit 1
-grep baz.test stdout && Exit 1
-grep '^# PASS: *1$' stdout
-grep '^# FAIL: *1$' stdout
-grep '^# ERROR: *1$' stdout
-
 $MAKE clean
 $MAKE check TESTS=baz > stdout && { cat stdout; Exit 1; }
 cat stdout
diff --git a/t/tap-recheck-logs.sh b/t/tap-lazy-check.sh
similarity index 77%
rename from t/tap-recheck-logs.sh
rename to t/tap-lazy-check.sh
index 36a59ca7a..de92c3d6b 100755
--- a/t/tap-recheck-logs.sh
+++ b/t/tap-lazy-check.sh
@@ -14,8 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# TAP support:
-# - RECHECK_LOGS
+# TAP support: AM_LAZY_CHECK
 
 am_parallel_tests=yes
 . ./defs || Exit 1
@@ -47,7 +46,7 @@ Bail out!
 END
 
 # Even the tests that are not re-run should contribute to the testsuite
-# summary when obtained by "make check RECHECK_LOGS=".
+# summary when obtained by "make check AM_LAZY_CHECK=yes".
 grep_summary ()
 {
   grep '^# TOTAL: *4$' stdout
@@ -66,7 +65,7 @@ test -f baz.log
 
 rm -f foo.log bar.log
 
-$MAKE RECHECK_LOGS= check > stdout && { cat stdout; Exit 1; }
+$MAKE AM_LAZY_CHECK=yes check > stdout && { cat stdout; Exit 1; }
 cat stdout
 test -f foo.log
 test -f bar.log
@@ -81,7 +80,7 @@ touch foo.test
 # We re-run only a successful test, but the tests that failed in the
 # previous run should still be taken into account, and cause an overall
 # failure.
-$MAKE RECHECK_LOGS= check > stdout && { cat stdout; Exit 1; }
+$MAKE AM_LAZY_CHECK=yes check > stdout && { cat stdout; Exit 1; }
 cat stdout
 grep '^PASS: foo\.test 1$' stdout
 grep '^PASS: foo\.test 2$' stdout
@@ -91,7 +90,7 @@ grep_summary
 
 $sleep
 touch zardoz
-$MAKE RECHECK_LOGS= check > stdout && { cat stdout; Exit 1; }
+$MAKE AM_LAZY_CHECK=yes check > stdout && { cat stdout; Exit 1; }
 cat stdout
 grep '^ERROR: baz\.test' stdout
 $EGREP '(foo|bar)\.test' stdout && Exit 1
@@ -100,20 +99,9 @@ grep_summary
 
 # Now, explicitly retry with all test logs already updated, and ensure
 # that the summary is still displayed.
-$MAKE RECHECK_LOGS= check > stdout && { cat stdout; Exit 1; }
+$MAKE AM_LAZY_CHECK=yes check > stdout && { cat stdout; Exit 1; }
 cat stdout
 $EGREP '(foo|bar|baz)\.test' stdout && Exit 1
 grep_summary
 
-# The following should re-run foo.test (and only foo.test), even if its
-# log file is up-to-date.
-: > older
-$MAKE RECHECK_LOGS=foo.log check > stdout && { cat stdout; Exit 1; }
-cat stdout
-grep '^PASS: foo\.test 1$' stdout
-grep '^PASS: foo\.test 2$' stdout
-grep 'ba[rz]\.test' stdout && Exit 1
-is_newest foo.log older
-grep_summary
-
 :
diff --git a/t/test-trs-recover.sh b/t/test-trs-recover.sh
index de3b77191..25cb36368 100755
--- a/t/test-trs-recover.sh
+++ b/t/test-trs-recover.sh
@@ -154,7 +154,7 @@ test -f baz.trs
 rm -f foo.trs
 update_stamp
 touch bar.test
-$MAKE RECHECK_LOGS= check >stdout || { cat stdout; Exit 1; }
+$MAKE AM_LAZY_CHECK=yes check >stdout || { cat stdout; Exit 1; }
 cat stdout
 # Check that make has updated what it needed to, but no more.
 test -f foo.trs