+2009-04-15 Eric Blake <ebb9@byu.net>
+
+ Teach AT_CHECK about hard failures.
+ * lib/autotest/general.m4 (AT_INIT) <at_fn_check_skip>
+ <at_fn_check_status, at_fn_group_postprocess>: Handle hard
+ failures.
+ * doc/autoconf.texi (Writing Testsuites) <AT_CHECK>: Document
+ AT_CHECK_NOESCAPE and exit status 99.
+ * NEWS: Likewise.
+ * tests/autotest.at (Hard fail, Cleanup): New tests.
+
2009-04-14 Eric Blake <ebb9@byu.net>
Fix yesterday's regression in AS_IF.
proper m4 quoting. For shell comments, this is a new feature; for
non-shell comments, this fixes a regression introduced in 2.63b.
+** The macro AT_CHECK now understands the concept of hard failure. If
+ a test exits with an unexpected status 99, cleanup actions for the
+ test are inhibited and the test is treated as a failure regardless
+ of AT_XFAIL_IF.
+
+** The autotest macro AT_CHECK_NOESCAPE is now documented.
+
** The following documented m4sugar macros are new:
m4_default_nblank m4_default_nblank_quoted m4_ifblank m4_ifnblank
be a single shell word that expands into a single file name.
@end defmac
-@defmac AT_CHECK (@var{commands}, @dvar{status, 0}, @dvar{stdout, }, @
- @dvar{stderr, }, @ovar{run-if-fail}, @ovar{run-if-pass})
+@defmac AT_CHECK (@var{commands}, @dvar{status, 0}, @ovar{stdout}, @
+ @ovar{stderr}, @ovar{run-if-fail}, @ovar{run-if-pass})
+@defmacx AT_CHECK_NOESCAPE (@var{commands}, @dvar{status, 0}, @ovar{stdout}, @
+ @ovar{stderr}, @ovar{run-if-fail}, @ovar{run-if-pass})
@atindex{CHECK}
+@atindex{CHECK_NOESCAPE}
Execute a test by performing given shell @var{commands}. These commands
should normally exit with @var{status}, while producing expected
@var{stdout} and @var{stderr} contents. If @var{commands} exit with
-status 77, then the whole test group is skipped. Otherwise, if this test
+unexpected status 77, then the rest of the test group is skipped. If
+@var{commands} exit with unexpected status 99, then the test group is
+immediately failed. Otherwise, if this test
fails, run shell commands @var{run-if-fail} or, if this test passes, run shell
commands @var{run-if-pass}.
This macro must be invoked in between @code{AT_SETUP} and @code{AT_CLEANUP}.
-@c Previously, we had this:
-@c The @var{commands} @emph{must not} redirect the standard output, nor the
-@c standard error.
-@c to prevent trigerring the double redirect bug on Ultrix, see
-@c `File Descriptors'. This was too restricting, and Ultrix is pretty
-@c much dead, so we dropped the limitation; the obvious workaround on
-@c Ultrix is to use a working shell there.
-
-If @var{status}, or @var{stdout}, or @var{stderr} is @samp{ignore}, then
-the corresponding value is not checked.
-
-The special value @samp{expout} for @var{stdout} means the expected
-output of the @var{commands} is the content of the file @file{expout}.
-If @var{stdout} is @samp{stdout}, then the standard output of the
-@var{commands} is available for further tests in the file @file{stdout}.
-Similarly for @var{stderr} with @samp{experr} and @samp{stderr}.
+If @var{status} is the literal @samp{ignore}, then the corresponding
+exit status is not checked, except for the special cases of 77 (skip)
+and 99 (hard failure). The existence of hard failures allows one to
+mark a test as an expected failure with @code{AT_XFAIL_IF} because a
+feature has not yet been implemented, but to still distinguish between
+gracefully handling the missing feature and dumping core. A hard
+failure also inhibits post-test actions in @var{run-if-fail}.
+
+If the value of the @var{stdout} or @var{stderr} parameter is one of the
+literals in the following table, then the test treats the output
+according to the rules of that literal. Otherwise, the value of the
+parameter is treated as text that must exactly match the output given by
+@var{commands} on standard out and standard error (including an empty
+parameter for no output); any differences are captured in the testsuite
+log and the test is failed. The difference between @code{AT_CHECK} and
+@code{AT_CHECK_NOESCAPE} is that only the latter performs shell
+expansions on comparison text given in the @var{stdout} and @var{stderr}
+arguments.
+
+@table @samp
+@item ignore
+The content of the output is ignored, but still captured in the test
+group log (if the test group later fails, the test group log is then
+copied into the overall testsuite log). This is valid for both
+@var{stdout} and @var{stderr}.
+
+@item stdout
+For the @var{stdout} parameter, capture the content of standard output
+to both the file @file{stdout} and the test group log. Subsequent
+commands in the test group can then post-process the file. This action
+is often used when it is desired to use @command{grep} to look for a
+substring in the output, or when the output must be post-processed to
+normalize error messages into a common form.
+
+@item stderr
+Like @samp{stdout}, except that it only works for the @var{stderr}
+parameter, and the standard error capture file will be named
+@file{stderr}.
+
+@item expout
+For the @var{stdout} parameter, compare standard output contents with
+the previously created file @file{expout}, and list any differences in
+the testsuite log.
+
+@item experr
+Like @samp{expout}, except that it only works for the @var{stderr}
+parameter, and the standard error contents are compared with
+@file{experr}.
+@end table
@end defmac
exit 1
}
-AS_FUNCTION_DESCRIBE([at_fn_check_skip], [EXIT-CODE],
-[Check whether EXIT-CODE is the special exit code 77, and if so exit the shell
-with that same exit code.])
+AS_FUNCTION_DESCRIBE([at_fn_check_skip], [EXIT-CODE LINE],
+[Check whether EXIT-CODE is a special exit code (77 or 99), and if so exit
+the test group subshell with that same exit code. Use LINE in any report
+about test failure.])
at_fn_check_skip ()
{
case $[1] in
+ 99) echo 99 > "$at_status_file"; at_failed=:
+ AS_ECHO(["$[2]: hard failure"]); exit 99;;
77) echo 77 > "$at_status_file"; exit 77;;
esac
}
AS_FUNCTION_DESCRIBE([at_fn_check_status], [EXPECTED EXIT-CODE LINE],
-[Check whether EXIT-CODE is the expected exit code, and if so do nothing.
-Otherwise, if it is 77 exit the shell with that same exit code; if it is
-anything else print an error message and fail the test.])
+[Check whether EXIT-CODE is the EXPECTED exit code, and if so do nothing.
+Otherwise, if it is 77 or 99, exit the test group subshell with that same
+exit code; if it is anything else print an error message referring to LINE,
+and fail the test.])
at_fn_check_status ()
{
dnl This order ensures that we don't `skip' if we are precisely checking
-dnl $? = 77.
+dnl $? = 77 or $? = 99.
case $[2] in
$[1] ) ;;
77) echo 77 > "$at_status_file"; exit 77;;
+ 99) echo 99 > "$at_status_file"; at_failed=:
+ AS_ECHO(["$[3]: hard failure"]); exit 99;;
*) AS_ECHO(["$[3]: exit code was $[2], expected $[1]"])
at_failed=:;;
esac
report this failure to <AT_PACKAGE_BUGREPORT>.
_ATEOF
AS_ECHO(["$at_setup_line"]) >"$at_check_line_file"
- at_xfail=no at_status=99
+ at_status=99
fi
$at_verbose AS_ECHO_N(["$at_group. $at_setup_line: "])
AS_ECHO_N(["$at_group. $at_setup_line: "]) >> "$at_group_log"
case $at_xfail:$at_status in
+ *:99)
+ at_msg='FAILED ('`cat "$at_check_line_file"`')'
+ at_res=fail
+ at_errexit=$at_errexit_p
+ ;;
yes:0)
at_msg="UNEXPECTED PASS"
at_res=xpass
[], [], [], [], [],
[AT_CHECK([grep skipped micro-suite.log], [], [ignore])])
+AT_CHECK_AT_TEST([Hard fail],
+ [AT_CHECK([exit 99])
+ AT_CLEANUP
+ AT_SETUP([another test])
+ AT_XFAIL_IF([:])
+ AT_CHECK([exit 99])],
+ [], [1], [], [ignore], [],
+ [AT_CHECK([grep '2 failed unexpectedly' micro-suite.log], [], [ignore])
+ AT_CHECK([grep ok micro-suite.log], [1])])
+
AT_CHECK_AT_TEST([Syntax error],
[AT_CHECK([:])
AT_CLEANUP
AT_CHECK([grep "1 .* inhibited subsequent" stderr], [], [ignore])],
[--errexit])
+AT_CHECK_AT_TEST([Cleanup],
+ [AT_CHECK([test ! -f cleanup.success && test ! -f cleanup.failure])
+ AT_CHECK_NOESCAPE([exit $value], [ignore], [$output],
+ [], [touch cleanup.failure], [touch cleanup.success])],
+ [], [], [], [],
+ [AT_KEYWORDS([AT@&t@_CHECK_NOESCAPE])
+ output=; export output],
+ [AT_CHECK([test -d micro-suite.dir/1])
+ AT_CHECK([test -f micro-suite.dir/1/cleanup.success])
+ AT_CHECK([test ! -f micro-suite.dir/1/cleanup.failure])
+
+ AT_CHECK([$CONFIG_SHELL ./micro-suite -d value=1], [], [ignore])
+ AT_CHECK([test -f micro-suite.dir/1/cleanup.success])
+ AT_CHECK([test ! -f micro-suite.dir/1/cleanup.failure])
+
+ AT_CHECK([$CONFIG_SHELL ./micro-suite -d value=1 output=mismatch],
+ [1], [ignore], [ignore])
+ AT_CHECK([test ! -f micro-suite.dir/1/cleanup.success])
+ AT_CHECK([test -f micro-suite.dir/1/cleanup.failure])
+
+ AT_CHECK([$CONFIG_SHELL ./micro-suite -d value=77], [], [ignore])
+ AT_CHECK([test ! -f micro-suite.dir/1/cleanup.success])
+ AT_CHECK([test ! -f micro-suite.dir/1/cleanup.failure])
+
+ AT_CHECK([$CONFIG_SHELL ./micro-suite -d value=99], [1], [ignore], [ignore])
+ AT_CHECK([test ! -f micro-suite.dir/1/cleanup.success])
+ AT_CHECK([test ! -f micro-suite.dir/1/cleanup.failure])
+ ], [-d value=0])
+
## ----------------------------------------------------- ##
## Newlines and command substitutions in test commands. ##
## ----------------------------------------------------- ##
projects / thirdparty / autoconf.git / commitdiff
