From: Eric Blake Date: Wed, 5 Mar 2008 20:39:23 +0000 (-0700) Subject: Improve error messages for common testsuite bugs. X-Git-Tag: v2.62~41 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=21ba57f84006eb01b24093282b2ff14c5bda636e;p=thirdparty%2Fautoconf.git Improve error messages for common testsuite bugs. * lib/autotest/general.m4 (_AT_DEFINE_INIT, _AT_DEFINE_SETUP): New macros for defining order-enforced macros. (AT_INIT, AT_SETUP, AT_CLEANUP, AT_BANNER, AT_XFAIL_IF) (AT_CAPTURE_FILE, AT_DATA, AT_CHECK, AT_CHECK_NOESCAPE): Add error messages when order violations are detected. * tests/autotest.at (AT_CHECK_AT_SYNTAX): New helper macro. (AT_SETUP without AT_INIT, AT_BANNER without AT_INIT) (AT_CLEANUP without AT_INIT, Missing AT_CLEANUP) (AT_CHECK without AT_SETUP, AT_DATA without AT_SETUP) (AT_XFAIL_IF without AT_DATA, AT_KEYWORDS without AT_SETUP, (AT_CLEANUP without AT_SETUP, AT_BANNER inside AT_SETUP) (AT_SETUP inside AT_SETUP, Multiple AT_INIT) (Banner-only test suite): New tests. Reported by Christopher Hulbert. Signed-off-by: Eric Blake --- diff --git a/ChangeLog b/ChangeLog index 94168a06..c6cc1e75 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,21 @@ 2008-03-11 Eric Blake + Improve error messages for common testsuite bugs. + * lib/autotest/general.m4 (_AT_DEFINE_INIT, _AT_DEFINE_SETUP): New + macros for defining order-enforced macros. + (AT_INIT, AT_SETUP, AT_CLEANUP, AT_BANNER, AT_XFAIL_IF) + (AT_CAPTURE_FILE, AT_DATA, AT_CHECK, AT_CHECK_NOESCAPE): Add error + messages when order violations are detected. + * tests/autotest.at (AT_CHECK_AT_SYNTAX): New helper macro. + (AT_SETUP without AT_INIT, AT_BANNER without AT_INIT) + (AT_CLEANUP without AT_INIT, Missing AT_CLEANUP) + (AT_CHECK without AT_SETUP, AT_DATA without AT_SETUP) + (AT_XFAIL_IF without AT_DATA, AT_KEYWORDS without AT_SETUP, + (AT_CLEANUP without AT_SETUP, AT_BANNER inside AT_SETUP) + (AT_SETUP inside AT_SETUP, Multiple AT_INIT) + (Banner-only test suite): New tests. + Reported by Christopher Hulbert. + Tweak m4_wrap to force FIFO or LIFO semantics. * lib/m4sugar/m4sugar.m4 (m4_wrap): Override M4 implementation. (m4_wrap_lifo, _m4_wrap): New macros. diff --git a/lib/autotest/general.m4 b/lib/autotest/general.m4 index 2f930b5b..c4cff41b 100644 --- a/lib/autotest/general.m4 +++ b/lib/autotest/general.m4 @@ -181,19 +181,37 @@ m4_define([_AT_NORMALIZE_TEST_GROUP_NUMBER], done' ]) +# _AT_DEFINE_INIT(NAME, [DEFINITION]) +# ----------------------------------- +# Define macro NAME to die if invoked prior to AT_INIT, and to DEFINITION +# after AT_INIT. +m4_define([_AT_DEFINE_INIT], +[m4_define($@)m4_pushdef([$1], [m4_fatal([$1: missing AT_INIT detected])])dnl +m4_append([_AT_DEFINE_INIT_LIST], [[$1]], [,])]) + +# _AT_DEFINE_SETUP(NAME, [DEFINITION]) +# ----------------------------------- +# Define macro NAME to die if invoked outside AT_SETUP/AT_CLEANUP, and +# to DEFINITION otherwise. +m4_define([_AT_DEFINE_SETUP], +[m4_define([$1], [m4_ifndef([AT_ingroup], + [m4_fatal([$1: missing AT_SETUP detected])])$2])]) + # AT_INIT([TESTSUITE-NAME]) # ------------------------- # Begin test suite. m4_define([AT_INIT], -[m4_pattern_forbid([^_?AT_]) +[m4_pushdef([AT_INIT], [m4_fatal([$0: invoked multiple times])]) +m4_pattern_forbid([^_?AT_]) m4_pattern_allow([^_AT_T_EOF$]) m4_define([AT_TESTSUITE_NAME], - m4_defn([AT_PACKAGE_STRING])[ test suite]m4_ifval([$1], [: $1])) + m4_defn([AT_PACKAGE_STRING])[ test suite]m4_ifval([$1], [m4_expand([: $1])])) m4_define([AT_ordinal], 0) m4_define([AT_banner_ordinal], 0) m4_define([AT_groups_all], []) m4_define([AT_help_all], []) +m4_foreach([AT_name], [_AT_DEFINE_INIT_LIST], [m4_popdef(m4_defn([AT_name]))]) m4_wrap([_AT_FINISH]) AS_INIT[]dnl m4_divert_push([DEFAULTS])dnl @@ -404,7 +422,9 @@ esac] # Whether -C is in effect. at_change_dir=false m4_divert_pop([DEFAULTS])dnl -m4_define([_AT_FINISH], [m4_divert_text([DEFAULTS], +m4_define([_AT_FINISH], +[m4_ifdef([AT_ingroup], [m4_fatal([missing AT_CLEANUP detected])])dnl +m4_divert_text([DEFAULTS], [ # List of the tested programs. at_tested='m4_ifdef([AT_tested], @@ -1401,8 +1421,10 @@ $1])])# AT_COPYRIGHT # --------------------- # Start a group of related tests, all to be executed in the same subshell. # The group is testing what DESCRIPTION says. -m4_define([AT_SETUP], -[m4_ifdef([AT_keywords], [m4_undefine([AT_keywords])]) +_AT_DEFINE_INIT([AT_SETUP], +[m4_ifdef([AT_ingroup], [m4_fatal([$0: nested AT_SETUP detected])], + [m4_define([AT_ingroup])]) +m4_ifdef([AT_keywords], [m4_undefine([AT_keywords])]) m4_define([AT_capture_files], []) m4_define([AT_line], AT_LINE) m4_define([AT_xfail], [at_xfail=no]) @@ -1426,7 +1448,7 @@ m4_divert_push([TEST_SCRIPT])dnl # ----------------------------- # Set up the test to be expected to fail if SHELL-EXPRESSION evaluates to # true (exitcode = 0). -m4_define([AT_XFAIL_IF], +_AT_DEFINE_SETUP([AT_XFAIL_IF], [dnl dnl Try to limit the amount of conditionals that we emit. m4_case([$1], @@ -1442,7 +1464,7 @@ m4_case([$1], # --------------------- # Declare a list of keywords associated to the current test group. # The list is stored in lower case, since the -k option is case-insensitive. -m4_define([AT_KEYWORDS], +_AT_DEFINE_SETUP([AT_KEYWORDS], [m4_append_uniq_w([AT_keywords], m4_tolower([[$1]]))]) @@ -1450,7 +1472,7 @@ m4_define([AT_KEYWORDS], # --------------------- # If the current test group does not behave as expected, save the contents of # FILE in the test suite log. -m4_define([AT_CAPTURE_FILE], +_AT_DEFINE_SETUP([AT_CAPTURE_FILE], [m4_append_uniq([AT_capture_files], ["$1"], [ \ ])]) @@ -1458,8 +1480,10 @@ m4_define([AT_CAPTURE_FILE], # AT_CLEANUP # ---------- # Complete a group of related tests. -m4_define([AT_CLEANUP], -[m4_append([AT_help_all], +_AT_DEFINE_INIT([AT_CLEANUP], +[m4_ifdef([AT_ingroup], [m4_undefine([AT_ingroup])], + [m4_fatal([$0: missing AT_SETUP detected])])dnl +m4_append([AT_help_all], m4_defn([AT_ordinal]);m4_defn([AT_line]);m4_defn([AT_description]);dnl m4_ifdef([AT_keywords], [m4_defn([AT_keywords])]); )dnl @@ -1484,8 +1508,9 @@ m4_divert_pop([TEST_GROUPS])dnl Back to KILL. # Start a category of related test groups. If multiple groups are executed, # output TEXT as a banner without any shell expansion, prior to any test # from the category. If TEXT is empty, no banner is printed. -m4_define([AT_BANNER], -[m4_define([AT_banner_ordinal], m4_incr(AT_banner_ordinal)) +_AT_DEFINE_INIT([AT_BANNER], +[m4_ifdef([AT_ingroup], [m4_fatal([$0: nested AT_SETUP detected])])dnl +m4_define([AT_banner_ordinal], m4_incr(AT_banner_ordinal)) m4_divert_text([BANNERS], [@%:@ Banner AT_banner_ordinal. AT_LINE @%:@ Category starts at test group m4_incr(AT_ordinal). @@ -1499,7 +1524,7 @@ at_banner_text_[]AT_banner_ordinal="AS_ESCAPE([$1])"])dnl # an end of line. # This macro is not robust to active symbols in CONTENTS *on purpose*. # If you don't want CONTENTS to be evaluated, quote it twice. -m4_define([AT_DATA], +_AT_DEFINE_SETUP([AT_DATA], [cat >$1 <<'_ATEOF' $2[]_ATEOF ]) @@ -1545,7 +1570,7 @@ $2[]_ATEOF # out, since most shells when tracing include subshell traces in stderr. # This may cause spurious failures when the test suite is run with `-x'. # -m4_define([AT_CHECK], +_AT_DEFINE_SETUP([AT_CHECK], [_AT_CHECK([$1],[$2],[$3],[$4],[$5],[$6],1)]) # AT_CHECK_NOESCAPE(COMMANDS, [STATUS = 0], STDOUT, STDERR, @@ -1553,7 +1578,7 @@ m4_define([AT_CHECK], # --------------------------------------------------------- # Like AT_CHECK, but do not AS_ESCAPE shell metacharacters in the STDOUT # and STDERR arguments before running the comparison. -m4_define([AT_CHECK_NOESCAPE], +_AT_DEFINE_SETUP([AT_CHECK_NOESCAPE], [_AT_CHECK([$1],[$2],[$3],[$4],[$5],[$6])]) diff --git a/tests/autotest.at b/tests/autotest.at index fea0adc9..3ba3ebfb 100644 --- a/tests/autotest.at +++ b/tests/autotest.at @@ -81,7 +81,20 @@ AT_INIT([artificial test suite]) AT_SETUP([my only test]) $2 AT_CLEANUP -]], m4_shift2($@))]) +]], m4_shift2($@))]) # AT_CHECK_AT_TEST + +# AT_CHECK_AT_SYNTAX(TITLE, SUITE, MESSAGE) +# ----------------------------------------- +# Create a test named TITLE that tries compiling SUITE with syntax +# errors with autotest. Expect a failed compilation, and grep for +# MESSAGE occuring within the error output. +m4_define([AT_CHECK_AT_SYNTAX], +[AT_SETUP([$1]) +AT_CHECK_AT_PREP([micro-suite], [$2], [1], [], [stderr]) +AT_CHECK([grep '$3' stderr], [0], [ignore]) +AT_CLEANUP +])# AT_CHECK_AT_SYNTAX + # AT_NO_CMDSUBST # -------------- @@ -109,12 +122,86 @@ AT_CHECK_AT([Empty test suite], [[AT_INIT([empty test suite]) ]]) +AT_CHECK_AT([Banner-only test suite], +[[AT_INIT([empty test suite]) +AT_BANNER([banner]) +]]) + # Next level of emptiness. AT_CHECK_AT_TEST([Empty test], []) # And finally, an empty check should not cause a syntax error. AT_CHECK_AT_TEST([Empty check], [AT_CHECK]) +# Check for sensible error messages for common bugs. +AT_CHECK_AT_SYNTAX([AT@&t@_SETUP without AT@&t@_INIT], +[[AT_SETUP([only test]) +AT_CHECK([:]) +AT_CLEANUP +]], [AT@&t@_SETUP: missing AT@&t@_INIT detected]) + +AT_CHECK_AT_SYNTAX([AT@&t@_BANNER without AT@&t@_INIT], +[[AT_BANNER([just a banner]) +]], [AT@&t@_BANNER: missing AT@&t@_INIT detected]) + +AT_CHECK_AT_SYNTAX([AT@&t@_CLEANUP without AT@&t@_INIT], +[[AT_CLEANUP +]], [AT@&t@_CLEANUP: missing AT@&t@_INIT detected]) + +AT_CHECK_AT_SYNTAX([Missing AT@&t@_CLEANUP], +[[AT_INIT([incomplete test suite]) +AT_SETUP([only test]) +AT_CHECK([:]) +]], [missing AT@&t@_CLEANUP detected]) + +AT_CHECK_AT_SYNTAX([AT@&t@_CHECK without AT@&t@_SETUP], +[[AT_INIT([incomplete test suite]) +AT_CHECK([:]) +]], [AT@&t@_CHECK: missing AT@&t@_SETUP detected]) + +AT_CHECK_AT_SYNTAX([AT@&t@_DATA without AT@&t@_SETUP], +[[AT_INIT([incomplete test suite]) +AT_DATA([file]) +]], [AT@&t@_DATA: missing AT@&t@_SETUP detected]) + +AT_CHECK_AT_SYNTAX([AT@&t@_XFAIL_IF without AT@&t@_SETUP], +[[AT_INIT([incomplete test suite]) +AT_XFAIL_IF([:]) +]], [AT@&t@_XFAIL_IF: missing AT@&t@_SETUP detected]) + +AT_CHECK_AT_SYNTAX([AT@&t@_KEYWORDS without AT@&t@_SETUP], +[[AT_INIT([incomplete test suite]) +AT_KEYWORDS([keyword]) +]], [AT@&t@_KEYWORDS: missing AT@&t@_SETUP detected]) + +AT_CHECK_AT_SYNTAX([AT@&t@_CLEANUP without AT@&t@_SETUP], +[[AT_INIT([incomplete test suite]) +AT_CLEANUP +]], [AT@&t@_CLEANUP: missing AT@&t@_SETUP detected]) + +AT_CHECK_AT_SYNTAX([AT@&t@_BANNER inside AT@&t@_SETUP], +[[AT_INIT([incomplete test suite]) +AT_SETUP([only test]) +AT_BANNER([banner]) +AT_CHECK([:]) +AT_CLEANUP +]], [AT@&t@_BANNER: nested AT@&t@_SETUP detected]) + +AT_CHECK_AT_SYNTAX([AT@&t@_SETUP inside AT@&t@_SETUP], +[[AT_INIT([incomplete test suite]) +AT_SETUP([only test]) + AT_SETUP([nested test]) + AT_CHECK([:]) + AT_CLEANUP +AT_CHECK([:]) +AT_CLEANUP +]], [AT@&t@_SETUP: nested AT@&t@_SETUP detected]) + +AT_CHECK_AT_SYNTAX([Multiple AT@&t@_INIT], +[[AT_INIT([[suite, take one]]) +AT_INIT([repeat]) +]], [AT@&t@_INIT: invoked multiple times]) + # Check for tested programs. autoconf should only appear once. AT_CHECK_AT([Tested programs], [[AT_INIT([programs test suite])