Fix LAZY_TEST_SUITE handling and $(TEST_SUITE_LOG) recreation.

* lib/am/check.am (check-TESTS): Expand `$(TEST_LOGS)' only once
in the rule command, for systems with low command line limits.
Remove $(TEST_SUITE_LOG) even in LAZY_TEST_SUITE mode.
($(TEST_SUITE_LOG)): Always recreate $(TEST_SUITE_LOG).  Mention
lazy mode in the summary output.
* tests/parallel-tests.test: Test LAZY_TEST_SUITE semantics.

Signed-off-by: Ralf Wildenhues <Ralf.Wildenhues@gmx.de>

diff --git a/ChangeLog b/ChangeLog
index 38af3a8dfb1189c81fc98181baec43e1cbb26c87..659167c75ec7d1bfe0d2f9985c51105353e45c70 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2009-03-11  Ralf Wildenhues  <Ralf.Wildenhues@gmx.de>
+
+       Fix LAZY_TEST_SUITE handling and $(TEST_SUITE_LOG) recreation.
+       * lib/am/check.am (check-TESTS): Expand `$(TEST_LOGS)' only once
+       in the rule command, for systems with low command line limits.
+       Remove $(TEST_SUITE_LOG) even in LAZY_TEST_SUITE mode.
+       ($(TEST_SUITE_LOG)): Always recreate $(TEST_SUITE_LOG).  Mention
+       lazy mode in the summary output.
+       * tests/parallel-tests.test: Test LAZY_TEST_SUITE semantics.
+
 2009-03-10  Ralf Wildenhues  <Ralf.Wildenhues@gmx.de>
 
        Fix parallel-tests with empty $(TESTS) for BSD make.

diff --git a/lib/am/check.am b/lib/am/check.am
index 5a2de7d1a1158d72224b2f47af74830134cda6a6..fcb9add2a980f72673067aec17d703f005b35082 100644 (file)
--- a/lib/am/check.am
+++ b/lib/am/check.am
@@ -184,23 +184,26 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
            msg="$$msg($$skip tests were not run).  ";                  \
          fi;                                                           \
        fi;                                                             \
+       {                                                               \
+         echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" |       \
+           $(am__rst_title);                                           \
+         echo "$$msg";                                                 \
+         echo;                                                         \
+         echo ".. contents:: :depth: 2";                               \
+         echo;                                                         \
+         for f in $$list; do                                           \
+           read line < $$f;                                            \
+           case $$line in                                              \
+             SKIP:*|PASS:*|XFAIL:*);;                                  \
+             *) echo; cat $$f;;                                        \
+           esac;                                                       \
+         done;                                                         \
+       } >$(TEST_SUITE_LOG).tmp;                                       \
+       mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG);                     \
+       if test -n '$(LAZY_TEST_SUITE)'; then                           \
+         msg="$${msg}(tests were rerun lazily).  ";                    \
+       fi;                                                             \
        if test "$$failures" -ne 0; then                                \
-         {                                                             \
-           echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" |     \
-             $(am__rst_title);                                         \
-           echo "$$msg";                                               \
-           echo;                                                       \
-           echo ".. contents:: :depth: 2";                             \
-           echo;                                                       \
-           for f in $$list; do                                         \
-             read line < $$f;                                          \
-             case $$line in                                            \
-               SKIP:*|PASS:*|XFAIL:*);;                                \
-               *) echo; cat $$f;;                                      \
-             esac;                                                     \
-           done;                                                       \
-         } >$(TEST_SUITE_LOG).tmp;                                     \
-         mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG);                   \
          msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG).  ";              \
          if test -n "$(PACKAGE_BUGREPORT)"; then                       \
            msg="$${msg}Please report to $(PACKAGE_BUGREPORT).  ";      \
@@ -219,10 +222,20 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
 
 # Run all the tests.
 check-TESTS:
-       @if test -z '$(LAZY_TEST_SUITE)'                                \
-           && test -n "$(TEST_SUITE_LOG)$(TEST_LOGS)"; then            \
-         rm -f $(TEST_SUITE_LOG) $(TEST_LOGS); \
+## Expand $(TEST_LOGS) only once, to avoid exceeding line length limits.
+       @list='$(TEST_LOGS)'; if test -z '$(LAZY_TEST_SUITE)'           \
+         && test -n "$$list"; then                                     \
+         rm -f $$list;                                                 \
        fi
+## We always have to remove TEST_SUITE_LOG, to ensure its rule is run
+## in any case even in lazy mode: otherwise, if no test needs rerunning,
+## or a prior run plus reruns all happen within the same timestamp
+## (can happen with a prior `make TESTS=<subset>'),
+## then we get no log output.
+## OTOH, this means that, in the rule for `$(TEST_SUITE_LOG)', we
+## cannot use `$?' to compute the set of lazily rerun tests, lest
+## we rely on .PHONY to work portably.
+       @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
        @set_logs=; if test "X$(TEST_LOGS)" = X.log; then               \
          set_logs=TEST_LOGS=;                                          \
        fi;                                                             \

diff --git a/tests/Makefile.in b/tests/Makefile.in
index 1c80daa6d7cf1cc586e84b07fbf21ae2ec649f1d..6a293a8597934de860ae0613c9e1b25e227c6fd5 100644 (file)
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -1053,23 +1053,26 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
            msg="$$msg($$skip tests were not run).  ";                  \
          fi;                                                           \
        fi;                                                             \
+       {                                                               \
+         echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" |       \
+           $(am__rst_title);                                           \
+         echo "$$msg";                                                 \
+         echo;                                                         \
+         echo ".. contents:: :depth: 2";                               \
+         echo;                                                         \
+         for f in $$list; do                                           \
+           read line < $$f;                                            \
+           case $$line in                                              \
+             SKIP:*|PASS:*|XFAIL:*);;                                  \
+             *) echo; cat $$f;;                                        \
+           esac;                                                       \
+         done;                                                         \
+       } >$(TEST_SUITE_LOG).tmp;                                       \
+       mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG);                     \
+       if test -n '$(LAZY_TEST_SUITE)'; then                           \
+         msg="$${msg}(tests were rerun lazily).  ";                    \
+       fi;                                                             \
        if test "$$failures" -ne 0; then                                \
-         {                                                             \
-           echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" |     \
-             $(am__rst_title);                                         \
-           echo "$$msg";                                               \
-           echo;                                                       \
-           echo ".. contents:: :depth: 2";                             \
-           echo;                                                       \
-           for f in $$list; do                                         \
-             read line < $$f;                                          \
-             case $$line in                                            \
-               SKIP:*|PASS:*|XFAIL:*);;                                \
-               *) echo; cat $$f;;                                      \
-             esac;                                                     \
-           done;                                                       \
-         } >$(TEST_SUITE_LOG).tmp;                                     \
-         mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG);                   \
          msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG).  ";              \
          if test -n "$(PACKAGE_BUGREPORT)"; then                       \
            msg="$${msg}Please report to $(PACKAGE_BUGREPORT).  ";      \
@@ -1088,10 +1091,11 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
 
 # Run all the tests.
 check-TESTS:
-       @if test -z '$(LAZY_TEST_SUITE)'                                \
-           && test -n "$(TEST_SUITE_LOG)$(TEST_LOGS)"; then            \
-         rm -f $(TEST_SUITE_LOG) $(TEST_LOGS); \
+       @list='$(TEST_LOGS)'; if test -z '$(LAZY_TEST_SUITE)'           \
+         && test -n "$$list"; then                                     \
+         rm -f $$list;                                                 \
        fi
+       @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
        @set_logs=; if test "X$(TEST_LOGS)" = X.log; then               \
          set_logs=TEST_LOGS=;                                          \
        fi;                                                             \

diff --git a/tests/parallel-tests.test b/tests/parallel-tests.test
index 2482f30b6013a2eae75a6f7824110cff0a915605..e7ebd3db5baad0c8c9fa535dfd32f96a0918c2b8 100755 (executable)
--- a/tests/parallel-tests.test
+++ b/tests/parallel-tests.test
@@ -20,6 +20,7 @@
 # - TEST_SUITE_LOG
 # - dependencies between tests
 # - DISABLE_HARD_ERRORS
+# - LAZY_TEST_SUITE
 
 . ./defs-p || Exit 1
 
@@ -101,6 +102,30 @@ test -f bar.log
 test ! -f foo.log
 test -f mylog.log
 
+# Upon a lazy rerun, foo.test should be run, but the others shouldn't.
+# Note that the lazy rerun still exits with a failure, due to the previous
+# test failures.
+# Note that the previous test and this one taken together expose the timing
+# issue that requires the check-TESTS rule to always remove TEST_SUITE_LOG
+# before running the tests lazily.
+env LAZY_TEST_SUITE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+test -f foo.log
+grep foo.test stdout
+grep bar.test stdout && Exit 1
+grep baz.test stdout && Exit 1
+grep '2.*tests.*failed' stdout
+grep 'lazily' stdout
+
+# Now, explicitly retry with all test logs already updated, and ensure
+# that the summary is still displayed.
+env LAZY_TEST_SUITE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+grep foo.test stdout && Exit 1
+grep bar.test stdout && Exit 1
+grep baz.test stdout && Exit 1
+grep '2.*tests.*failed' stdout
+
 # Test VERBOSE.
 env VERBOSE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; }
 cat stdout