testsuite: reorganize tests on TAP plans

* tests/tap-skip-whole.test: Rename ...
* tests/tap-planskip.test: ... to this
* tests/tap-skip-whole-badexit.test: Rename ...
* tests/tap-planskip-badexit.test: ... to this.
* tests/tap-skip-whole-bailout.test: Rename ...
* tests/tap-planskip-bailout.test: ... to this.
* tests/tap-skip-whole-bailout.test: Rename ...
* tests/tap-planskip-later-errors.test: ... to this.
* tests/tap-skip-whole-unplanned.test: Rename ...
* tests/tap-planskip-unplanned.test: ... to this.
* tests/tap-skip-whole-whitespace.test: Rename ...
* tests/tap-planskip-whitespace.test: ... to this.
* tests/tap-skip-whole-continue.test: Rename ...
* tests/tap-planskip-and-logging.test: ... to this, extend to try
more cases, and fix heading comments.
* tests/tap-skip-whole-lastline.test: Rename ...
* tests/tap-planskip-late.test: ... to this, fix heading comments,
and remove some checks that are just a duplication of those already
performed in the previous script.
* tests/tap-results-with-planskip.test: Rename to ...
* tests/tap-planskip-unplanned-corner.test: ... this, and improve
hading comments.
* tests/tap-planskip-later-errors.test: New test.
* tests/Makefile.am (tap_with_common_setup_tests): Update.

diff --git a/ChangeLog b/ChangeLog
index 19ac010138aae944c14b4e7e9d0d5451fb62cc45..c596fe53c8c7a381bf3995150fee08433a37ba14 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,31 @@
+2011-08-12  Stefano Lattarini  <stefano.lattarini@gmail.com>
+
+       testsuite: reorganize tests on TAP plans
+       * tests/tap-skip-whole.test: Rename ...
+       * tests/tap-planskip.test: ... to this
+       * tests/tap-skip-whole-badexit.test: Rename ...
+       * tests/tap-planskip-badexit.test: ... to this.
+       * tests/tap-skip-whole-bailout.test: Rename ...
+       * tests/tap-planskip-bailout.test: ... to this.
+       * tests/tap-skip-whole-bailout.test: Rename ...
+       * tests/tap-planskip-later-errors.test: ... to this.
+       * tests/tap-skip-whole-unplanned.test: Rename ...
+       * tests/tap-planskip-unplanned.test: ... to this.
+       * tests/tap-skip-whole-whitespace.test: Rename ...
+       * tests/tap-planskip-whitespace.test: ... to this.
+       * tests/tap-skip-whole-continue.test: Rename ...
+       * tests/tap-planskip-and-logging.test: ... to this, extend to try
+       more cases, and fix heading comments.
+       * tests/tap-skip-whole-lastline.test: Rename ...
+       * tests/tap-planskip-late.test: ... to this, fix heading comments,
+       and remove some checks that are just a duplication of those already
+       performed in the previous script.
+       * tests/tap-results-with-planskip.test: Rename to ...
+       * tests/tap-planskip-unplanned-corner.test: ... this, and improve
+       hading comments.
+       * tests/tap-planskip-later-errors.test: New test.
+       * tests/Makefile.am (tap_with_common_setup_tests): Update.
+
 2011-08-12  Stefano Lattarini  <stefano.lattarini@gmail.com>
 
        coverage: more TAP driver consistency w.r.t. 'prove'

diff --git a/tests/Makefile.am b/tests/Makefile.am
index d1b294eddc46a31ad7e26ca1f305b6516dd28806..b44b0d5a19c558ea8c4b51dccad400c7fbbd0c40 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1174,16 +1174,17 @@ tap-plan.test \
 tap-plan-corner.test \
 tap-plan-errors.test \
 tap-plan-middle.test \
+tap-planskip.test \
+tap-planskip-late.test \
+tap-planskip-and-logging.test \
+tap-planskip-unplanned.test \
+tap-planskip-unplanned-corner.test \
+tap-planskip-whitespace.test \
+tap-planskip-badexit.test \
+tap-planskip-bailout.test \
+tap-planskip-later-errors.test \
 tap-realtime.test \
 tap-recheck-logs.test \
-tap-skip-whole-whitespace.test \
-tap-skip-whole.test \
-tap-skip-whole-lastline.test \
-tap-skip-whole-badexit.test \
-tap-skip-whole-bailout.test \
-tap-skip-whole-continue.test \
-tap-skip-whole-unplanned.test \
-tap-results-with-planskip.test \
 tap-todo-skip-together.test \
 tap-todo-skip-whitespace.test \
 tap-todo-skip.test \

diff --git a/tests/Makefile.in b/tests/Makefile.in
index 9c39e9b455a6552a619b26a0572009053bdf925b..d1366750d835725115bae88cf4370b07a80d0ed9 100644 (file)
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -1414,16 +1414,17 @@ tap-plan.test \
 tap-plan-corner.test \
 tap-plan-errors.test \
 tap-plan-middle.test \
+tap-planskip.test \
+tap-planskip-late.test \
+tap-planskip-and-logging.test \
+tap-planskip-unplanned.test \
+tap-planskip-unplanned-corner.test \
+tap-planskip-whitespace.test \
+tap-planskip-badexit.test \
+tap-planskip-bailout.test \
+tap-planskip-later-errors.test \
 tap-realtime.test \
 tap-recheck-logs.test \
-tap-skip-whole-whitespace.test \
-tap-skip-whole.test \
-tap-skip-whole-lastline.test \
-tap-skip-whole-badexit.test \
-tap-skip-whole-bailout.test \
-tap-skip-whole-continue.test \
-tap-skip-whole-unplanned.test \
-tap-results-with-planskip.test \
 tap-todo-skip-together.test \
 tap-todo-skip-whitespace.test \
 tap-todo-skip.test \

diff --git a/tests/tap-skip-whole-continue.test b/tests/tap-planskip-and-logging.test
similarity index 75%
rename from tests/tap-skip-whole-continue.test
rename to tests/tap-planskip-and-logging.test
index 274fe54f464ade6e6ea50fae4f25bcdae942d97a..621608be8b378f8fd3b12c857b582da130e9a3ae 100755 (executable)
--- a/tests/tap-skip-whole-continue.test
+++ b/tests/tap-planskip-and-logging.test
@@ -14,9 +14,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Basic TAP test protocol support:
-#  - parsing of input TAP stream continue after a "SKIP" plan has
-#    been seen.
+# TAP support:
+#  - interactions between "TAP plan with SKIP" and logging of earlier or
+#    later TAP or non-TAP text
 
 parallel_tests=yes
 . ./defs || Exit 1
@@ -31,6 +31,12 @@ a non-TAP line
 # a comment
 END
 
+cat > foo2.test <<END
+a non-TAP line 2
+# a comment 2
+1..0
+END
+
 cat > bar.test <<END
 # an early comment
 an early non-TAP line
@@ -40,20 +46,25 @@ an early non-TAP line
 a later non-TAP line
 END
 
-TESTS='foo.test bar.test' $MAKE -e check >stdout || { cat stdout; Exit 1; }
+TESTS='foo.test foo2.test bar.test' $MAKE -e check >stdout \
+  || { cat stdout; Exit 1; }
 cat stdout
 
-count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=2 error=0
+count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=3 error=0
 
 grep '^# foo\.test: a comment$' stdout
+grep '^# foo2\.test: a comment 2$' stdout
 grep '^# bar\.test: an early comment$' stdout
 grep '^# bar\.test: a later comment$' stdout
 
 cat foo.log
+cat foo2.log
 cat bar.log
 
 grep '^a non-TAP line$' foo.log
 grep '^# a comment$' foo.log
+grep '^a non-TAP line 2$' foo2.log
+grep '^# a comment 2$' foo2.log
 grep '^# an early comment' bar.log
 grep '^an early non-TAP line$' bar.log
 grep '^# a later comment' bar.log

diff --git a/tests/tap-skip-whole-badexit.test b/tests/tap-planskip-badexit.test
similarity index 100%
rename from tests/tap-skip-whole-badexit.test
rename to tests/tap-planskip-badexit.test

diff --git a/tests/tap-skip-whole-bailout.test b/tests/tap-planskip-bailout.test
similarity index 100%
rename from tests/tap-skip-whole-bailout.test
rename to tests/tap-planskip-bailout.test

diff --git a/tests/tap-skip-whole-lastline.test b/tests/tap-planskip-late.test
similarity index 68%
rename from tests/tap-skip-whole-lastline.test
rename to tests/tap-planskip-late.test
index 56bb4e2c6c705e06aede2c51df9e91666858ff95..c2cb0a83dfd8891e02a8a18b82f253f3349b4fc7 100755 (executable)
--- a/tests/tap-skip-whole-lastline.test
+++ b/tests/tap-planskip-late.test
@@ -15,17 +15,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # TAP support:
-#  - the special plan format to skip all the tests in a script can also
-#    be used on the last line, even if preceded by non-TAP output, TAP
-#    diagnostic, and even non-failing TAP test results.
+#  - the special "plan with SKIP" can also be used "late" in the TAP
+#    stream, i.e., preceded by non-TAP output or TAP diagnostic.
 
 parallel_tests=yes
 . ./defs || Exit 1
 
 . "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
 
-echo AM_TEST_LOG_DRIVER_FLAGS = --comments >> Makefile
-
 cat > foo.test <<END
 some non-TAP text, will be copied in the global log
 1..0 # SKIP from the last line
@@ -36,17 +33,11 @@ cat > bar.test <<END
 1..0
 END
 
-st=0
-TESTS='foo.test bar.test' $MAKE -e check >stdout || st=$?
+TESTS='foo.test bar.test' $MAKE -e check >stdout || { cat stdout; Exit 1; }
 cat stdout
-cat test-suite.log
-test $st -eq 0
 
 grep '^SKIP: foo\.test .* from the last line$' stdout
 grep '^SKIP: bar\.test$' stdout
-grep '^# bar\.test: some TAP diagnostic, will go to console$' stdout
-grep '^# some TAP diagnostic, will go to console$' test-suite.log
-grep '^some non-TAP text, will be copied in the global log$' test-suite.log
 test `grep -c ': .*\.test' stdout` -eq 2
 count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=2 error=0
 

diff --git a/tests/tap-planskip-later-errors.test b/tests/tap-planskip-later-errors.test
new file mode 100755 (executable)
index 0000000..eb2ee26
--- /dev/null
+++ b/tests/tap-planskip-later-errors.test
@@ -0,0 +1,40 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+#  - TAP errors following a "TAP plan with SKIP" are still diagnosed.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<END
+1..0
+a non-TAP line
+1..3
+1..5
+# a comment
+1..1
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=4 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=3
+test `grep -c '^ERROR: all\.test - multiple test plans' stdout` -eq 3
+
+:

diff --git a/tests/tap-results-with-planskip.test b/tests/tap-planskip-unplanned-corner.test
similarity index 95%
rename from tests/tap-results-with-planskip.test
rename to tests/tap-planskip-unplanned-corner.test
index fa4f06e1b27555be242a24ba298e1614e4aedd22..cf9f49eb3991892a7e3e1275def0a54bf0debca6 100755 (executable)
--- a/tests/tap-results-with-planskip.test
+++ b/tests/tap-planskip-unplanned-corner.test
@@ -16,7 +16,7 @@
 
 # TAP support:
 #  - test results seen in a TAP stream that has a "plan with SKIP" are
-#    flagged as errors.
+#    flagged as errors, even if all of them are "SKIP" results.
 
 parallel_tests=yes
 . ./defs || Exit 1

diff --git a/tests/tap-skip-whole-unplanned.test b/tests/tap-planskip-unplanned.test
similarity index 100%
rename from tests/tap-skip-whole-unplanned.test
rename to tests/tap-planskip-unplanned.test

diff --git a/tests/tap-skip-whole-whitespace.test b/tests/tap-planskip-whitespace.test
similarity index 100%
rename from tests/tap-skip-whole-whitespace.test
rename to tests/tap-planskip-whitespace.test

diff --git a/tests/tap-skip-whole.test b/tests/tap-planskip.test
similarity index 100%
rename from tests/tap-skip-whole.test
rename to tests/tap-planskip.test