From 320685333ede68b51e2c720941b4f90010268760 Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Fri, 12 Aug 2011 13:12:54 +0200 Subject: [PATCH] testsuite: reorganize tests on TAP plans * tests/tap-skip-whole.test: Rename ... * tests/tap-planskip.test: ... to this * tests/tap-skip-whole-badexit.test: Rename ... * tests/tap-planskip-badexit.test: ... to this. * tests/tap-skip-whole-bailout.test: Rename ... * tests/tap-planskip-bailout.test: ... to this. * tests/tap-skip-whole-bailout.test: Rename ... * tests/tap-planskip-later-errors.test: ... to this. * tests/tap-skip-whole-unplanned.test: Rename ... * tests/tap-planskip-unplanned.test: ... to this. * tests/tap-skip-whole-whitespace.test: Rename ... * tests/tap-planskip-whitespace.test: ... to this. * tests/tap-skip-whole-continue.test: Rename ... * tests/tap-planskip-and-logging.test: ... to this, extend to try more cases, and fix heading comments. * tests/tap-skip-whole-lastline.test: Rename ... * tests/tap-planskip-late.test: ... to this, fix heading comments, and remove some checks that are just a duplication of those already performed in the previous script. * tests/tap-results-with-planskip.test: Rename to ... * tests/tap-planskip-unplanned-corner.test: ... this, and improve hading comments. * tests/tap-planskip-later-errors.test: New test. * tests/Makefile.am (tap_with_common_setup_tests): Update. --- ChangeLog | 28 +++++++++++++ tests/Makefile.am | 17 ++++---- tests/Makefile.in | 17 ++++---- ...nue.test => tap-planskip-and-logging.test} | 21 +++++++--- ...badexit.test => tap-planskip-badexit.test} | 0 ...bailout.test => tap-planskip-bailout.test} | 0 ...e-lastline.test => tap-planskip-late.test} | 15 ++----- tests/tap-planskip-later-errors.test | 40 +++++++++++++++++++ ...est => tap-planskip-unplanned-corner.test} | 2 +- ...anned.test => tap-planskip-unplanned.test} | 0 ...pace.test => tap-planskip-whitespace.test} | 0 ...{tap-skip-whole.test => tap-planskip.test} | 0 12 files changed, 106 insertions(+), 34 deletions(-) rename tests/{tap-skip-whole-continue.test => tap-planskip-and-logging.test} (75%) rename tests/{tap-skip-whole-badexit.test => tap-planskip-badexit.test} (100%) rename tests/{tap-skip-whole-bailout.test => tap-planskip-bailout.test} (100%) rename tests/{tap-skip-whole-lastline.test => tap-planskip-late.test} (68%) create mode 100755 tests/tap-planskip-later-errors.test rename tests/{tap-results-with-planskip.test => tap-planskip-unplanned-corner.test} (95%) rename tests/{tap-skip-whole-unplanned.test => tap-planskip-unplanned.test} (100%) rename tests/{tap-skip-whole-whitespace.test => tap-planskip-whitespace.test} (100%) rename tests/{tap-skip-whole.test => tap-planskip.test} (100%) diff --git a/ChangeLog b/ChangeLog index 19ac01013..c596fe53c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,31 @@ +2011-08-12 Stefano Lattarini + + testsuite: reorganize tests on TAP plans + * tests/tap-skip-whole.test: Rename ... + * tests/tap-planskip.test: ... to this + * tests/tap-skip-whole-badexit.test: Rename ... + * tests/tap-planskip-badexit.test: ... to this. + * tests/tap-skip-whole-bailout.test: Rename ... + * tests/tap-planskip-bailout.test: ... to this. + * tests/tap-skip-whole-bailout.test: Rename ... + * tests/tap-planskip-later-errors.test: ... to this. + * tests/tap-skip-whole-unplanned.test: Rename ... + * tests/tap-planskip-unplanned.test: ... to this. + * tests/tap-skip-whole-whitespace.test: Rename ... + * tests/tap-planskip-whitespace.test: ... to this. + * tests/tap-skip-whole-continue.test: Rename ... + * tests/tap-planskip-and-logging.test: ... to this, extend to try + more cases, and fix heading comments. + * tests/tap-skip-whole-lastline.test: Rename ... + * tests/tap-planskip-late.test: ... to this, fix heading comments, + and remove some checks that are just a duplication of those already + performed in the previous script. + * tests/tap-results-with-planskip.test: Rename to ... + * tests/tap-planskip-unplanned-corner.test: ... this, and improve + hading comments. + * tests/tap-planskip-later-errors.test: New test. + * tests/Makefile.am (tap_with_common_setup_tests): Update. + 2011-08-12 Stefano Lattarini coverage: more TAP driver consistency w.r.t. 'prove' diff --git a/tests/Makefile.am b/tests/Makefile.am index d1b294edd..b44b0d5a1 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1174,16 +1174,17 @@ tap-plan.test \ tap-plan-corner.test \ tap-plan-errors.test \ tap-plan-middle.test \ +tap-planskip.test \ +tap-planskip-late.test \ +tap-planskip-and-logging.test \ +tap-planskip-unplanned.test \ +tap-planskip-unplanned-corner.test \ +tap-planskip-whitespace.test \ +tap-planskip-badexit.test \ +tap-planskip-bailout.test \ +tap-planskip-later-errors.test \ tap-realtime.test \ tap-recheck-logs.test \ -tap-skip-whole-whitespace.test \ -tap-skip-whole.test \ -tap-skip-whole-lastline.test \ -tap-skip-whole-badexit.test \ -tap-skip-whole-bailout.test \ -tap-skip-whole-continue.test \ -tap-skip-whole-unplanned.test \ -tap-results-with-planskip.test \ tap-todo-skip-together.test \ tap-todo-skip-whitespace.test \ tap-todo-skip.test \ diff --git a/tests/Makefile.in b/tests/Makefile.in index 9c39e9b45..d1366750d 100644 --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -1414,16 +1414,17 @@ tap-plan.test \ tap-plan-corner.test \ tap-plan-errors.test \ tap-plan-middle.test \ +tap-planskip.test \ +tap-planskip-late.test \ +tap-planskip-and-logging.test \ +tap-planskip-unplanned.test \ +tap-planskip-unplanned-corner.test \ +tap-planskip-whitespace.test \ +tap-planskip-badexit.test \ +tap-planskip-bailout.test \ +tap-planskip-later-errors.test \ tap-realtime.test \ tap-recheck-logs.test \ -tap-skip-whole-whitespace.test \ -tap-skip-whole.test \ -tap-skip-whole-lastline.test \ -tap-skip-whole-badexit.test \ -tap-skip-whole-bailout.test \ -tap-skip-whole-continue.test \ -tap-skip-whole-unplanned.test \ -tap-results-with-planskip.test \ tap-todo-skip-together.test \ tap-todo-skip-whitespace.test \ tap-todo-skip.test \ diff --git a/tests/tap-skip-whole-continue.test b/tests/tap-planskip-and-logging.test similarity index 75% rename from tests/tap-skip-whole-continue.test rename to tests/tap-planskip-and-logging.test index 274fe54f4..621608be8 100755 --- a/tests/tap-skip-whole-continue.test +++ b/tests/tap-planskip-and-logging.test @@ -14,9 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Basic TAP test protocol support: -# - parsing of input TAP stream continue after a "SKIP" plan has -# been seen. +# TAP support: +# - interactions between "TAP plan with SKIP" and logging of earlier or +# later TAP or non-TAP text parallel_tests=yes . ./defs || Exit 1 @@ -31,6 +31,12 @@ a non-TAP line # a comment END +cat > foo2.test < bar.test <stdout || { cat stdout; Exit 1; } +TESTS='foo.test foo2.test bar.test' $MAKE -e check >stdout \ + || { cat stdout; Exit 1; } cat stdout -count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=2 error=0 +count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=3 error=0 grep '^# foo\.test: a comment$' stdout +grep '^# foo2\.test: a comment 2$' stdout grep '^# bar\.test: an early comment$' stdout grep '^# bar\.test: a later comment$' stdout cat foo.log +cat foo2.log cat bar.log grep '^a non-TAP line$' foo.log grep '^# a comment$' foo.log +grep '^a non-TAP line 2$' foo2.log +grep '^# a comment 2$' foo2.log grep '^# an early comment' bar.log grep '^an early non-TAP line$' bar.log grep '^# a later comment' bar.log diff --git a/tests/tap-skip-whole-badexit.test b/tests/tap-planskip-badexit.test similarity index 100% rename from tests/tap-skip-whole-badexit.test rename to tests/tap-planskip-badexit.test diff --git a/tests/tap-skip-whole-bailout.test b/tests/tap-planskip-bailout.test similarity index 100% rename from tests/tap-skip-whole-bailout.test rename to tests/tap-planskip-bailout.test diff --git a/tests/tap-skip-whole-lastline.test b/tests/tap-planskip-late.test similarity index 68% rename from tests/tap-skip-whole-lastline.test rename to tests/tap-planskip-late.test index 56bb4e2c6..c2cb0a83d 100755 --- a/tests/tap-skip-whole-lastline.test +++ b/tests/tap-planskip-late.test @@ -15,17 +15,14 @@ # along with this program. If not, see . # TAP support: -# - the special plan format to skip all the tests in a script can also -# be used on the last line, even if preceded by non-TAP output, TAP -# diagnostic, and even non-failing TAP test results. +# - the special "plan with SKIP" can also be used "late" in the TAP +# stream, i.e., preceded by non-TAP output or TAP diagnostic. parallel_tests=yes . ./defs || Exit 1 . "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" -echo AM_TEST_LOG_DRIVER_FLAGS = --comments >> Makefile - cat > foo.test < bar.test <stdout || st=$? +TESTS='foo.test bar.test' $MAKE -e check >stdout || { cat stdout; Exit 1; } cat stdout -cat test-suite.log -test $st -eq 0 grep '^SKIP: foo\.test .* from the last line$' stdout grep '^SKIP: bar\.test$' stdout -grep '^# bar\.test: some TAP diagnostic, will go to console$' stdout -grep '^# some TAP diagnostic, will go to console$' test-suite.log -grep '^some non-TAP text, will be copied in the global log$' test-suite.log test `grep -c ': .*\.test' stdout` -eq 2 count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=2 error=0 diff --git a/tests/tap-planskip-later-errors.test b/tests/tap-planskip-later-errors.test new file mode 100755 index 000000000..eb2ee261c --- /dev/null +++ b/tests/tap-planskip-later-errors.test @@ -0,0 +1,40 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: +# - TAP errors following a "TAP plan with SKIP" are still diagnosed. + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +cat > all.test <stdout && { cat stdout; Exit 1; } +cat stdout + +count_test_results total=4 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=3 +test `grep -c '^ERROR: all\.test - multiple test plans' stdout` -eq 3 + +: diff --git a/tests/tap-results-with-planskip.test b/tests/tap-planskip-unplanned-corner.test similarity index 95% rename from tests/tap-results-with-planskip.test rename to tests/tap-planskip-unplanned-corner.test index fa4f06e1b..cf9f49eb3 100755 --- a/tests/tap-results-with-planskip.test +++ b/tests/tap-planskip-unplanned-corner.test @@ -16,7 +16,7 @@ # TAP support: # - test results seen in a TAP stream that has a "plan with SKIP" are -# flagged as errors. +# flagged as errors, even if all of them are "SKIP" results. parallel_tests=yes . ./defs || Exit 1 diff --git a/tests/tap-skip-whole-unplanned.test b/tests/tap-planskip-unplanned.test similarity index 100% rename from tests/tap-skip-whole-unplanned.test rename to tests/tap-planskip-unplanned.test diff --git a/tests/tap-skip-whole-whitespace.test b/tests/tap-planskip-whitespace.test similarity index 100% rename from tests/tap-skip-whole-whitespace.test rename to tests/tap-planskip-whitespace.test diff --git a/tests/tap-skip-whole.test b/tests/tap-planskip.test similarity index 100% rename from tests/tap-skip-whole.test rename to tests/tap-planskip.test -- 2.47.2