tap: non-zero exit status after "Bail out!" should not be reported

This new behaviour is more consistent with that of the `prove'
utility.  Also, experience with the use of TAP in the Automake's
own test suite has shown that reporting non-zero exit status from
a script after it has issued a "Bail out!" directive is mostly
redundant, more introducing noise rather than conveying useful
information.

* lib/tap-driver (finish): Inline the part of it that was getting
the script exit status ...
(main): ... in here.
* tests/tap-bailout-badexit.test: New test.
* tests/Makefile.am (tap_with_common_setup_tests): Update.

diff --git a/ChangeLog b/ChangeLog
index 4ea0ee7633dd2c50b841cb68d297925913d2dfbe..9a1d6011edb6184a9d2798a6bf8bf4acabcd22a0 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+2011-08-12  Stefano Lattarini  <stefano.lattarini@gmail.com>
+
+       tap: non-zero exit status after "Bail out!" should not be reported
+       This new behaviour is more consistent with that of the `prove'
+       utility.  Also, experience with the use of TAP in the Automake's
+       own test suite has shown that reporting non-zero exit status from
+       a script after it has issued a "Bail out!" directive is mostly
+       redundant, more introducing noise rather than conveying useful
+       information.
+       * lib/tap-driver (finish): Inline the part of it that was getting
+       the script exit status ...
+       (main): ... in here.
+       * tests/tap-bailout-badexit.test: New test.
+       * tests/Makefile.am (tap_with_common_setup_tests): Update.
+
 2011-08-11  Stefano Lattarini  <stefano.lattarini@gmail.com>
 
        tap: a "plan with SKIP" after test results is an error

diff --git a/lib/tap-driver b/lib/tap-driver
index a02557dbf11556d2acb301a364b559f0f4b11355..b7a8ffbe33c5e99fbb7810a7b652c648458eb355 100755 (executable)
--- a/lib/tap-driver
+++ b/lib/tap-driver
@@ -235,10 +235,6 @@ sub get_test_exit_message ()
 
 sub finish ()
 {
-  if (!$cfg{"ignore-exit"} and my $msg = get_test_exit_message)
-    {
-      testsuite_error $msg;
-    }
   write_test_results;
   close LOG or die "closing $log_file: $!\n";
   exit 0;
@@ -444,6 +440,10 @@ sub main (@)
       testsuite_error (sprintf "too %s tests run (expected %d, got %d)",
                                $bad_amount, $planned, $run);
     }
+  if (!$cfg{"ignore-exit"} and my $msg = get_test_exit_message)
+    {
+      testsuite_error $msg;
+    }
   finish;
 }
 

diff --git a/tests/Makefile.am b/tests/Makefile.am
index ae76516c98c1f209fbc4f9962f1fe155bf36f5bd..cf78cb649e18a79d9c3424033a4943aea31bb541 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1144,6 +1144,7 @@ testsuite-summary-count-many.log: extract-testsuite-summary
 tap_with_common_setup_tests = \
 tap-autonumber.test \
 tap-bailout.test \
+tap-bailout-badexit.test \
 tap-color.test \
 tap-deps.test \
 tap-diagnostic.test \

diff --git a/tests/Makefile.in b/tests/Makefile.in
index d43abe98a2ad035886f0fbdc2c02c2831bfa8bfe..39d1da07c9f45e6953818925872e370006fd55a4 100644 (file)
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -1384,6 +1384,7 @@ $(parallel_tests)
 tap_with_common_setup_tests = \
 tap-autonumber.test \
 tap-bailout.test \
+tap-bailout-badexit.test \
 tap-color.test \
 tap-deps.test \
 tap-diagnostic.test \

diff --git a/tests/tap-bailout-badexit.test b/tests/tap-bailout-badexit.test
new file mode 100755 (executable)
index 0000000..059394c
--- /dev/null
+++ b/tests/tap-bailout-badexit.test
@@ -0,0 +1,65 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Basic TAP test protocol support:
+#  - A "Bail out!" directive causes the driver to ignore the exit
+#    status of the test script.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+tests='exit.test exit127.test sighup.test sigterm.test'
+
+cat > Makefile.am <<END
+TESTS = $tests
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > exit.test << 'END'
+#!/bin/sh
+echo 'Bail out!'
+exit 1
+END
+
+cat > exit127.test << 'END'
+#!/bin/sh
+echo 'Bail out!'
+exit 127
+END
+
+cat > sighup.test << 'END'
+#!/bin/sh
+echo 'Bail out!'
+kill -1 $$
+END
+
+cat > sigterm.test << 'END'
+#!/bin/sh
+echo 'Bail out!'
+kill -15 $$
+END
+
+chmod a+x $tests
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=4 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=4
+for tst in $tests; do grep "^ERROR: $tst - Bail out!" stdout; done
+$EGREP "ERROR: .*(exit.*status|terminat.*signal)" stdout && Exit 1
+
+: