tap: improve granularity for tests on problematic TAP messages

* tests/tap-message-0.test: Break up into ...
* tests/tap-msg0-result.test, tests/tap-msg0-directive.test,
tests/tap-msg0-planskip.test, tests/tap-msg0-misc.test,
tests/tap-msg0-bailout.test: ... these new tests, and extend
a little.
* tests/Makefile.am (tap_with_common_setup_tests): Update.

diff --git a/ChangeLog b/ChangeLog
index 985cc86af6eec5db3ac90ce4969de72e676733f5..4d83efbca30cfb9fef828ca1d9e9645d948862de 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2011-08-17  Stefano Lattarini  <stefano.lattarini@gmail.com>
+
+       tap: improve granularity for tests on problematic TAP messages
+       * tests/tap-message-0.test: Break up into ...
+       * tests/tap-msg0-result.test, tests/tap-msg0-directive.test,
+       tests/tap-msg0-planskip.test, tests/tap-msg0-misc.test,
+       tests/tap-msg0-bailout.test: ... these new tests, and extend
+       a little.
+       * tests/Makefile.am (tap_with_common_setup_tests): Update.
+
 2011-08-17  Stefano Lattarini  <stefano.lattarini@gmail.com>
 
        tap: correctly handle string "0" in TAP messages

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 174f07bdfef5c2769a63bb7c3871af8bc28aa628..853831d7f573adbbb831b0bc80dc747e2f0bca83 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1161,9 +1161,13 @@ tap-global-log.test \
 tap-global-result.test \
 tap-html.test \
 tap-log.test \
+tap-msg0-result.test \
+tap-msg0-directive.test \
+tap-msg0-planskip.test \
+tap-msg0-bailout.test \
+tap-msg0-misc.test \
 tap-merge-stdout-stderr.test \
 tap-no-merge-stdout-stderr.test \
-tap-message-0.test \
 tap-no-disable-hard-error.test \
 tap-no-spurious-summary.test \
 tap-no-spurious.test \

diff --git a/tests/Makefile.in b/tests/Makefile.in
index b014a8d8d4694b7efda7b960ec00ef8087ae9b52..a1c152fb02a665dd485fea931f6e81ee27d936c2 100644 (file)
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -1402,9 +1402,13 @@ tap-global-log.test \
 tap-global-result.test \
 tap-html.test \
 tap-log.test \
+tap-msg0-result.test \
+tap-msg0-directive.test \
+tap-msg0-planskip.test \
+tap-msg0-bailout.test \
+tap-msg0-misc.test \
 tap-merge-stdout-stderr.test \
 tap-no-merge-stdout-stderr.test \
-tap-message-0.test \
 tap-no-disable-hard-error.test \
 tap-no-spurious-summary.test \
 tap-no-spurious.test \

diff --git a/tests/tap-msg0-bailout.test b/tests/tap-msg0-bailout.test
new file mode 100755 (executable)
index 0000000..6aa6d60
--- /dev/null
+++ b/tests/tap-msg0-bailout.test
@@ -0,0 +1,38 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+#  - literal strings "0" and "0.0" as a test description in the message
+#    of a "plan with skip" TAP plan
+# generally true!
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+echo 'Bail out! 0' > a.test
+echo 'Bail out! 0.0' > b.test
+
+TESTS='a.test b.test' $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=2
+
+grep '^ERROR: a.test - Bail out! 0$' stdout
+grep '^ERROR: b.test - Bail out! 0\.0$' stdout
+
+:

diff --git a/tests/tap-msg0-directive.test b/tests/tap-msg0-directive.test
new file mode 100755 (executable)
index 0000000..6ec63f7
--- /dev/null
+++ b/tests/tap-msg0-directive.test
@@ -0,0 +1,41 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+#  - literal string "0" as a TODO or SKIP message
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test << 'END'
+1..3
+ok 1 # TODO 0
+not ok 2 # TODO 0
+ok 3 # SKIP 0
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=0 fail=0 xpass=1 xfail=1 skip=1 error=0
+
+grep '^XPASS: all\.test 1 # TODO 0$' stdout
+grep '^XFAIL: all\.test 2 # TODO 0$' stdout
+grep '^SKIP: all\.test 3 # SKIP 0$' stdout
+
+:

diff --git a/tests/tap-msg0-misc.test b/tests/tap-msg0-misc.test
new file mode 100755 (executable)
index 0000000..c9399bf
--- /dev/null
+++ b/tests/tap-msg0-misc.test
@@ -0,0 +1,82 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+#  - literal "0" and "0.0" in a test description and a TODO/SKIP message
+#    at the same time
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test << 'END'
+1..14
+
+ok 1 0
+ok 2 0.0
+
+ok 3 0   # TODO 0
+ok 4 0.0 # TODO 0
+ok 5 0   # TODO 0.0
+ok 6 0.0 # TODO 0.0
+
+not ok 7 0   # TODO 0
+not ok 8 0.0 # TODO 0
+not ok 9 0   # TODO 0.0
+not ok 10 0.0 # TODO 0.0
+
+ok 11 0   # SKIP 0
+ok 12 0.0 # SKIP 0
+ok 13 0   # SKIP 0.0
+ok 14 0.0 # SKIP 0.0
+
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=14 pass=2 fail=0 xpass=4 xfail=4 skip=4 error=0
+
+sed '/^ *$/d' > exp << 'END'
+
+PASS: all.test 1 0
+PASS: all.test 2 0.0
+
+XPASS: all.test 3 0 # TODO 0
+XPASS: all.test 4 0.0 # TODO 0
+XPASS: all.test 5 0 # TODO 0.0
+XPASS: all.test 6 0.0 # TODO 0.0
+
+XFAIL: all.test 7 0 # TODO 0
+XFAIL: all.test 8 0.0 # TODO 0
+XFAIL: all.test 9 0 # TODO 0.0
+XFAIL: all.test 10 0.0 # TODO 0.0
+
+SKIP: all.test 11 0 # SKIP 0
+SKIP: all.test 12 0.0 # SKIP 0
+SKIP: all.test 13 0 # SKIP 0.0
+SKIP: all.test 14 0.0 # SKIP 0.0
+
+END
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:

diff --git a/tests/tap-msg0-planskip.test b/tests/tap-msg0-planskip.test
new file mode 100755 (executable)
index 0000000..b8c3d6c
--- /dev/null
+++ b/tests/tap-msg0-planskip.test
@@ -0,0 +1,37 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+#  - literal strings "0" and "0.0" as the reason of the skip in a "TAP
+#    plan with skip" (i.e., "1..0 # SKIP ...").
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+echo '1..0 # SKIP 0' > a.test
+echo '1..0 # SKIP 0.0' > b.test
+
+TESTS='a.test b.test' $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=2 error=0
+
+grep '^SKIP: a.test - 0$' stdout
+grep '^SKIP: b.test - 0\.0$' stdout
+
+:

diff --git a/tests/tap-message-0.test b/tests/tap-msg0-result.test
similarity index 54%
rename from tests/tap-message-0.test
rename to tests/tap-msg0-result.test
index 87b24925725ba0c51d11cce7adda7cdac09a2f6c..86259e5ea13640051aae85e4309b736cdccf322a 100755 (executable)
--- a/tests/tap-message-0.test
+++ b/tests/tap-msg0-result.test
@@ -15,8 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # TAP support:
-#  - having "0" as a test description (or TODO or SKIP or "Bail out!"
-#    message) should be supported
+#  - the string "0" as a test description
 
 parallel_tests=yes
 . ./defs || Exit 1
@@ -27,32 +26,32 @@ cat > all.test << 'END'
 1..10
 ok 1 0
 ok - 0
-not ok 3 0 # TODO
+not ok 3 0
+not ok - 0
+ok 5 0 # TODO
+ok - 0 # TODO
+not ok 7 0 # TODO
 not ok - 0 # TODO
-ok 5 0 # SKIP
+ok 9 0 # SKIP
 ok - 0 # SKIP
-not ok 7 # TODO 0
-ok 8 # SKIP 0
-not ok 9 0 # TODO 0
-ok 10 0 # SKIP 0
 END
 
-$MAKE check >stdout || { cat stdout; Exit 1; }
+$MAKE check >stdout && { cat stdout; Exit 1; }
 cat stdout
 
-count_test_results total=10 pass=2 fail=0 xpass=0 xfail=4 skip=4 error=0
+count_test_results total=10 pass=2 fail=2 xpass=2 xfail=2 skip=2 error=0
 
 cat > exp << 'END'
 PASS: all.test 1 0
 PASS: all.test 2 - 0
-XFAIL: all.test 3 0 # TODO
-XFAIL: all.test 4 - 0 # TODO
-SKIP: all.test 5 0 # SKIP
-SKIP: all.test 6 - 0 # SKIP
-XFAIL: all.test 7 # TODO 0
-SKIP: all.test 8 # SKIP 0
-XFAIL: all.test 9 0 # TODO 0
-SKIP: all.test 10 0 # SKIP 0
+FAIL: all.test 3 0
+FAIL: all.test 4 - 0
+XPASS: all.test 5 0 # TODO
+XPASS: all.test 6 - 0 # TODO
+XFAIL: all.test 7 0 # TODO
+XFAIL: all.test 8 - 0 # TODO
+SKIP: all.test 9 0 # SKIP
+SKIP: all.test 10 - 0 # SKIP
 END
 
 $FGREP ': all.test' stdout > got
@@ -61,21 +60,4 @@ cat exp
 cat got
 diff exp got
 
-echo '1..0 # SKIP 0' > all.test
-$MAKE check >stdout || { cat stdout; Exit 1; }
-cat stdout
-
-count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
-
-grep '^SKIP: all.test - 0' stdout
-
-echo 'Bail out! 0' > all.test
-$MAKE check >stdout && { cat stdout; Exit 1; }
-cat stdout
-
-count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
-
-grep '^ERROR: all.test - Bail out! 0' stdout
-
-
 :