New Autotest testsuite option --recheck.

* lib/autotest/general.m4 (AT_INIT): New variable $at_recheck.
Escape hyphen in $at_dir early.  Accept command line switch
--recheck.  Set $at_suite_log early, based on --directory
switch; with --recheck, include the list of FAILed and XPASSed
tests from old testsuite.log file in $at_groups.  Document
--recheck in --help output.
* tests/autotest.at (recheck): New test.
* doc/autoconf.texi (testsuite Invocation): Document --recheck.
* NEWS: Update.

Signed-off-by: Ralf Wildenhues <Ralf.Wildenhues@gmx.de>

diff --git a/ChangeLog b/ChangeLog
index a495513e34797dec22a0b4e74b6bbcb56240b0e8..84aa1d07bfb379179952bd09628cd14fc7ce4c7c 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+2010-06-14  Ralf Wildenhues  <Ralf.Wildenhues@gmx.de>
+
+       New Autotest testsuite option --recheck.
+       * lib/autotest/general.m4 (AT_INIT): New variable $at_recheck.
+       Escape hyphen in $at_dir early.  Accept command line switch
+       --recheck.  Set $at_suite_log early, based on --directory
+       switch; with --recheck, include the list of FAILed and XPASSed
+       tests from old testsuite.log file in $at_groups.  Document
+       --recheck in --help output.
+       * tests/autotest.at (recheck): New test.
+       * doc/autoconf.texi (testsuite Invocation): Document --recheck.
+       * NEWS: Update.
+
 2010-06-14  Karl Berry  <karl@freefriends.org>  (tiny change)
 
        Clarify comment about old system.

diff --git a/NEWS b/NEWS
index dfe8cd7796fa0a772beeb39b27c260aa6b454f0c..f1fdefba56f8164c4fc67c7d869304ee73364d8b 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -21,6 +21,9 @@ GNU Autoconf NEWS - User visible changes.
 
 ** AS_SET_CATFILE accepts nonliterals in its variable name argument now.
 
+** Autotest testsuites accept an option --recheck to rerun tests that
+   failed or passed unexpectedly during the last non-debug testsuite run.
+
 * Major changes in Autoconf 2.65 (2009-11-21) [stable]
   Released by Eric Blake, based on git versions 2.64.*.
 

diff --git a/doc/autoconf.texi b/doc/autoconf.texi
index 6be99aa20671d675d8708583ba5714e543c8a519..ca0d85cfd23107372c286f2a785b3b5c492c6de4 100644 (file)
--- a/doc/autoconf.texi
+++ b/doc/autoconf.texi
@@ -23805,6 +23805,10 @@ log (in order to preserve supposedly existing full log file).  This is
 the default for debugging scripts, but it can also be useful to debug
 the testsuite itself.
 
+@item --recheck
+Add to the selection all test groups that failed or passed unexpectedly
+during the last non-debugging test run.
+
 @item --trace
 @itemx -x
 Trigger shell tracing of the test groups.

diff --git a/lib/autotest/general.m4 b/lib/autotest/general.m4
index ac8f523902492e870ad97a62979a64387713aa77..1e38abe7c4d93af06f720f68c7e11d532a431a55 100644 (file)
--- a/lib/autotest/general.m4
+++ b/lib/autotest/general.m4
@@ -403,6 +403,8 @@ at_list_p=false
 at_clean=false
 # Test groups to run
 at_groups=
+# Whether to rerun failed tests.
+at_recheck=
 # Whether a write failure occurred
 at_write_fail=0
 
@@ -558,6 +560,9 @@ do
     --directory=* )
        at_change_dir=:
        at_dir=$at_optarg
+       if test x- = "x$at_dir" ; then
+         at_dir=./-
+       fi
        ;;
 
     # Parallel execution.
@@ -606,6 +611,9 @@ do
        `
        AS_VAR_APPEND([at_groups], ["$at_groups_selected "])
        ;;
+    --recheck)
+       at_recheck=:
+       ;;
 m4_divert_pop([PARSE_ARGS])dnl
 dnl Process *=* last to allow for user specified --option=* type arguments.
 m4_divert_push([PARSE_ARGS_END])dnl
@@ -633,10 +641,25 @@ done
 # Verify our last option didn't require an argument
 AS_IF([test -n "$at_prev"], [AS_ERROR([`$at_prev' requires an argument])])
 
+# The file containing the suite.
+at_suite_log=$at_dir/$as_me.log
+
 # Selected test groups.
-if test -z "$at_groups"; then
+if test -z "$at_groups$at_recheck"; then
   at_groups=$at_groups_all
 else
+  if test -n "$at_recheck" && test -r "$at_suite_log"; then
+    at_oldfails=`sed -n ['
+      /^Failed tests:$/,/^Skipped tests:$/{
+       s/^[ ]*\([1-9][0-9]*\):.*/\1/p
+      }
+      /^Unexpected passes:$/,/^## Detailed failed tests/{
+       s/^[ ]*\([1-9][0-9]*\):.*/\1/p
+      }
+      /^## Detailed failed tests/q
+      '] "$at_suite_log" | tr "$as_nl" ' '`
+    AS_VAR_APPEND([at_groups], ["$at_oldfails"])
+  fi
   # Sort the tests, removing duplicates.
   at_groups=`AS_ECHO(["$at_groups"]) | tr ' ' "$as_nl" | sort -nu`
 fi
@@ -686,6 +709,7 @@ Execution tuning:
   -k, --keywords=KEYWORDS
 [                 select the tests matching all the comma-separated KEYWORDS]
 [                 multiple \`-k' accumulate; prefixed \`!' negates a KEYWORD]
+      --recheck  select all tests that failed or passed unexpectedly last time
   -e, --errexit  abort as soon as a test fails; implies --debug
   -v, --verbose  force more detailed output
 [                 default for debugging scripts]
@@ -773,9 +797,6 @@ m4_divert_push([TESTS_BEGIN])dnl
 
 # Take any -C into account.
 if $at_change_dir ; then
-  if test x- = "x$at_dir" ; then
-    at_dir=./-
-  fi
   test x != "x$at_dir" && cd "$at_dir" \
     || AS_ERROR([unable to change directory])
   at_dir=`pwd`
@@ -840,7 +861,7 @@ m4_text_box([Directory structure.])
 # The directory the whole suite works in.
 # Should be absolute to let the user `cd' at will.
 at_suite_dir=$at_dir/$as_me.dir
-# The file containing the suite.
+# The file containing the suite ($at_dir might have changed since earlier).
 at_suite_log=$at_dir/$as_me.log
 # The directory containing helper files per test group.
 at_helper_dir=$at_suite_dir/at-groups

diff --git a/tests/autotest.at b/tests/autotest.at
index a09e2c1e0219a0b29dc2fe89137ae1aec0b0cdaf..ad38b7a965de1360055588fb01c716933a717fa0 100644 (file)
--- a/tests/autotest.at
+++ b/tests/autotest.at
@@ -890,6 +890,52 @@ AT_CHECK([$CONFIG_SHELL ./micro-suite -C nonesuch --help], [0], [ignore], [])
 ])
 
 
+# --recheck.
+AT_CHECK_AT_TEST([recheck],
+  [AT_CHECK([:])
+   AT_CLEANUP
+   AT_SETUP([failing test])
+   AT_CHECK([exit 1])
+   AT_CLEANUP
+   AT_SETUP([xpassing test])
+   AT_XFAIL_IF([:])
+   AT_CHECK([:])
+   AT_CLEANUP
+   AT_SETUP([xfailing test])
+   AT_XFAIL_IF([:])
+   AT_CHECK([exit 1])
+], [], [], [], [], [], [
+AT_CHECK([$CONFIG_SHELL ./micro-suite --recheck], [0], [stdout])
+AT_CHECK([grep "0 tests were successful" stdout], [0], [ignore])
+
+AT_CHECK([$CONFIG_SHELL ./micro-suite], [1], [ignore], [ignore])
+AT_CHECK([grep 'only test' micro-suite.log], [0], [ignore])
+AT_CHECK([$CONFIG_SHELL ./micro-suite --recheck], [1], [ignore], [ignore])
+AT_CHECK([grep 'only test' micro-suite.log], [1])
+AT_CHECK([grep 'xfailing' micro-suite.log], [1])
+AT_CHECK([grep 'failing test' micro-suite.log], [0], [ignore])
+AT_CHECK([grep 'xpassing test' micro-suite.log], [0], [ignore])
+
+AT_CHECK([$CONFIG_SHELL ./micro-suite --clean], [0])
+AT_CHECK([test -f micro-suite.log], [1])
+
+dnl check specifying a different relative path to run in.
+AT_CHECK([mkdir sub1])
+AT_CHECK([$CONFIG_SHELL ./micro-suite -C sub1], [1], [ignore], [ignore])
+AT_CHECK([test -f micro-suite.log], [1])
+AT_CHECK([test -f sub1/micro-suite.log], [0])
+AT_CHECK([$CONFIG_SHELL ./micro-suite -C sub1 --recheck --list], [0], [stdout])
+AT_CHECK([grep 'only test' stdout], [1])
+AT_CHECK([grep 'xfailing test' stdout], [1])
+AT_CHECK([grep 'failing test' stdout], [0], [ignore])
+AT_CHECK([grep 'xpassing test' stdout], [0], [ignore])
+AT_CHECK([$CONFIG_SHELL ./micro-suite -C sub1 --recheck], [1], [ignore],
+        [ignore])
+AT_CHECK([grep 'failing test' sub1/micro-suite.log], [0], [ignore])
+AT_CHECK([grep 'xpassing test' sub1/micro-suite.log], [0], [ignore])
+], [1 4])
+
+
 ## -------- ##
 ## Banners. ##
 ## -------- ##