From: Stefano Lattarini Date: Wed, 17 Aug 2011 16:30:12 +0000 (+0200) Subject: tap: improve granularity for tests on problematic TAP messages X-Git-Tag: ng-0.5a~89^2~99^2~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c8b66be05a6301e5f82ee7b3b7a87727f5cecd9f;p=thirdparty%2Fautomake.git tap: improve granularity for tests on problematic TAP messages * tests/tap-message-0.test: Break up into ... * tests/tap-msg0-result.test, tests/tap-msg0-directive.test, tests/tap-msg0-planskip.test, tests/tap-msg0-misc.test, tests/tap-msg0-bailout.test: ... these new tests, and extend a little. * tests/Makefile.am (tap_with_common_setup_tests): Update. --- diff --git a/ChangeLog b/ChangeLog index 985cc86af..4d83efbca 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,13 @@ +2011-08-17 Stefano Lattarini + + tap: improve granularity for tests on problematic TAP messages + * tests/tap-message-0.test: Break up into ... + * tests/tap-msg0-result.test, tests/tap-msg0-directive.test, + tests/tap-msg0-planskip.test, tests/tap-msg0-misc.test, + tests/tap-msg0-bailout.test: ... these new tests, and extend + a little. + * tests/Makefile.am (tap_with_common_setup_tests): Update. + 2011-08-17 Stefano Lattarini tap: correctly handle string "0" in TAP messages diff --git a/tests/Makefile.am b/tests/Makefile.am index 174f07bdf..853831d7f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1161,9 +1161,13 @@ tap-global-log.test \ tap-global-result.test \ tap-html.test \ tap-log.test \ +tap-msg0-result.test \ +tap-msg0-directive.test \ +tap-msg0-planskip.test \ +tap-msg0-bailout.test \ +tap-msg0-misc.test \ tap-merge-stdout-stderr.test \ tap-no-merge-stdout-stderr.test \ -tap-message-0.test \ tap-no-disable-hard-error.test \ tap-no-spurious-summary.test \ tap-no-spurious.test \ diff --git a/tests/Makefile.in b/tests/Makefile.in index b014a8d8d..a1c152fb0 100644 --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -1402,9 +1402,13 @@ tap-global-log.test \ tap-global-result.test \ tap-html.test \ tap-log.test \ +tap-msg0-result.test \ +tap-msg0-directive.test \ +tap-msg0-planskip.test \ +tap-msg0-bailout.test \ +tap-msg0-misc.test \ tap-merge-stdout-stderr.test \ tap-no-merge-stdout-stderr.test \ -tap-message-0.test \ tap-no-disable-hard-error.test \ tap-no-spurious-summary.test \ tap-no-spurious.test \ diff --git a/tests/tap-msg0-bailout.test b/tests/tap-msg0-bailout.test new file mode 100755 index 000000000..6aa6d60d4 --- /dev/null +++ b/tests/tap-msg0-bailout.test @@ -0,0 +1,38 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: +# - literal strings "0" and "0.0" as a test description in the message +# of a "plan with skip" TAP plan +# generally true! + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +echo 'Bail out! 0' > a.test +echo 'Bail out! 0.0' > b.test + +TESTS='a.test b.test' $MAKE -e check >stdout && { cat stdout; Exit 1; } +cat stdout + +count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=2 + +grep '^ERROR: a.test - Bail out! 0$' stdout +grep '^ERROR: b.test - Bail out! 0\.0$' stdout + +: diff --git a/tests/tap-msg0-directive.test b/tests/tap-msg0-directive.test new file mode 100755 index 000000000..6ec63f7d6 --- /dev/null +++ b/tests/tap-msg0-directive.test @@ -0,0 +1,41 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: +# - literal string "0" as a TODO or SKIP message + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +cat > all.test << 'END' +1..3 +ok 1 # TODO 0 +not ok 2 # TODO 0 +ok 3 # SKIP 0 +END + +$MAKE check >stdout && { cat stdout; Exit 1; } +cat stdout + +count_test_results total=3 pass=0 fail=0 xpass=1 xfail=1 skip=1 error=0 + +grep '^XPASS: all\.test 1 # TODO 0$' stdout +grep '^XFAIL: all\.test 2 # TODO 0$' stdout +grep '^SKIP: all\.test 3 # SKIP 0$' stdout + +: diff --git a/tests/tap-msg0-misc.test b/tests/tap-msg0-misc.test new file mode 100755 index 000000000..c9399bf32 --- /dev/null +++ b/tests/tap-msg0-misc.test @@ -0,0 +1,82 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: +# - literal "0" and "0.0" in a test description and a TODO/SKIP message +# at the same time + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +cat > all.test << 'END' +1..14 + +ok 1 0 +ok 2 0.0 + +ok 3 0 # TODO 0 +ok 4 0.0 # TODO 0 +ok 5 0 # TODO 0.0 +ok 6 0.0 # TODO 0.0 + +not ok 7 0 # TODO 0 +not ok 8 0.0 # TODO 0 +not ok 9 0 # TODO 0.0 +not ok 10 0.0 # TODO 0.0 + +ok 11 0 # SKIP 0 +ok 12 0.0 # SKIP 0 +ok 13 0 # SKIP 0.0 +ok 14 0.0 # SKIP 0.0 + +END + +$MAKE check >stdout && { cat stdout; Exit 1; } +cat stdout + +count_test_results total=14 pass=2 fail=0 xpass=4 xfail=4 skip=4 error=0 + +sed '/^ *$/d' > exp << 'END' + +PASS: all.test 1 0 +PASS: all.test 2 0.0 + +XPASS: all.test 3 0 # TODO 0 +XPASS: all.test 4 0.0 # TODO 0 +XPASS: all.test 5 0 # TODO 0.0 +XPASS: all.test 6 0.0 # TODO 0.0 + +XFAIL: all.test 7 0 # TODO 0 +XFAIL: all.test 8 0.0 # TODO 0 +XFAIL: all.test 9 0 # TODO 0.0 +XFAIL: all.test 10 0.0 # TODO 0.0 + +SKIP: all.test 11 0 # SKIP 0 +SKIP: all.test 12 0.0 # SKIP 0 +SKIP: all.test 13 0 # SKIP 0.0 +SKIP: all.test 14 0.0 # SKIP 0.0 + +END + +$FGREP ': all.test' stdout > got + +cat exp +cat got +diff exp got + +: diff --git a/tests/tap-msg0-planskip.test b/tests/tap-msg0-planskip.test new file mode 100755 index 000000000..b8c3d6c98 --- /dev/null +++ b/tests/tap-msg0-planskip.test @@ -0,0 +1,37 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: +# - literal strings "0" and "0.0" as the reason of the skip in a "TAP +# plan with skip" (i.e., "1..0 # SKIP ..."). + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +echo '1..0 # SKIP 0' > a.test +echo '1..0 # SKIP 0.0' > b.test + +TESTS='a.test b.test' $MAKE -e check >stdout || { cat stdout; Exit 1; } +cat stdout + +count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=2 error=0 + +grep '^SKIP: a.test - 0$' stdout +grep '^SKIP: b.test - 0\.0$' stdout + +: diff --git a/tests/tap-message-0.test b/tests/tap-msg0-result.test similarity index 54% rename from tests/tap-message-0.test rename to tests/tap-msg0-result.test index 87b249257..86259e5ea 100755 --- a/tests/tap-message-0.test +++ b/tests/tap-msg0-result.test @@ -15,8 +15,7 @@ # along with this program. If not, see . # TAP support: -# - having "0" as a test description (or TODO or SKIP or "Bail out!" -# message) should be supported +# - the string "0" as a test description parallel_tests=yes . ./defs || Exit 1 @@ -27,32 +26,32 @@ cat > all.test << 'END' 1..10 ok 1 0 ok - 0 -not ok 3 0 # TODO +not ok 3 0 +not ok - 0 +ok 5 0 # TODO +ok - 0 # TODO +not ok 7 0 # TODO not ok - 0 # TODO -ok 5 0 # SKIP +ok 9 0 # SKIP ok - 0 # SKIP -not ok 7 # TODO 0 -ok 8 # SKIP 0 -not ok 9 0 # TODO 0 -ok 10 0 # SKIP 0 END -$MAKE check >stdout || { cat stdout; Exit 1; } +$MAKE check >stdout && { cat stdout; Exit 1; } cat stdout -count_test_results total=10 pass=2 fail=0 xpass=0 xfail=4 skip=4 error=0 +count_test_results total=10 pass=2 fail=2 xpass=2 xfail=2 skip=2 error=0 cat > exp << 'END' PASS: all.test 1 0 PASS: all.test 2 - 0 -XFAIL: all.test 3 0 # TODO -XFAIL: all.test 4 - 0 # TODO -SKIP: all.test 5 0 # SKIP -SKIP: all.test 6 - 0 # SKIP -XFAIL: all.test 7 # TODO 0 -SKIP: all.test 8 # SKIP 0 -XFAIL: all.test 9 0 # TODO 0 -SKIP: all.test 10 0 # SKIP 0 +FAIL: all.test 3 0 +FAIL: all.test 4 - 0 +XPASS: all.test 5 0 # TODO +XPASS: all.test 6 - 0 # TODO +XFAIL: all.test 7 0 # TODO +XFAIL: all.test 8 - 0 # TODO +SKIP: all.test 9 0 # SKIP +SKIP: all.test 10 - 0 # SKIP END $FGREP ': all.test' stdout > got @@ -61,21 +60,4 @@ cat exp cat got diff exp got -echo '1..0 # SKIP 0' > all.test -$MAKE check >stdout || { cat stdout; Exit 1; } -cat stdout - -count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0 - -grep '^SKIP: all.test - 0' stdout - -echo 'Bail out! 0' > all.test -$MAKE check >stdout && { cat stdout; Exit 1; } -cat stdout - -count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1 - -grep '^ERROR: all.test - Bail out! 0' stdout - - :