+2011-05-11 Stefano Lattarini <stefano.lattarini@gmail.com>
+
+ check: document and test $(TEST_SUITE_LOG) overriding
+ * doc/automake.texi (Simple Tests using parallel-tests): Explain
+ how and why TEST_SUITE_LOG can be overridden at runtime.
+ * tests/parallel-tests-log-override-1.test: New test, check that
+ the newly documented idiom and similar usages are truly supported.
+ * tests/parallel-tests-log-override-2.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
2011-05-07 Stefano Lattarini <stefano.lattarini@gmail.com>
tests: fix spurious failure of txinfo21.test on FreeBSD
env TESTS="foo.test bar.test" make -e check
@end example
+Note however that the command above will unconditionally overwrite the
+@file{test-suite.log} file, thus clobbering the recorded results
+of any previous testsuite run. This might be undesirable for packages
+whose testsuite takes long time to execute. Luckily, this problem can
+easily be avoided by overriding also @code{TEST_SUITE_LOG} at runtime;
+for example,
+
+@example
+env TEST_SUITE_LOG=partial.log TESTS="..." make -e check
+@end example
+
+will write the result of the partial testsuite runs to the
+@file{partial.log}, without touching @file{test-suite.log}.
+
@item
You can set the @code{TEST_LOGS} variable. By default, this variable is
computed at @command{make} run time from the value of @code{TESTS} as
env TEST_LOGS="foo.log $*" make -e check
@end example
+The comments made above about @code{TEST_SUITE_LOG} overriding applies
+here too.
+
@item
@vindex RECHECK_LOGS
@cindex lazy test execution
parallel-tests10.test \
parallel-tests-unreadable-log.test \
parallel-tests-subdir.test \
+parallel-tests-log-override-1.test \
+parallel-tests-log-override-2.test \
+parallel-tests-log-override-recheck.test \
parse.test \
percent.test \
percent2.test \
parallel-tests10.test \
parallel-tests-unreadable-log.test \
parallel-tests-subdir.test \
+parallel-tests-log-override-1.test \
+parallel-tests-log-override-2.test \
+parallel-tests-log-override-recheck.test \
parse.test \
percent.test \
percent2.test \
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: runtime redefinition of $(TEST_SUITE_LOG).
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = pass.test skip.test xfail.test
+XFAIL_TESTS = xfail.test
+EXTRA_DIST = $(TESTS)
+END
+
+cat > pass.test <<'END'
+#! /bin/sh
+exit 0
+END
+
+cat > skip.test <<'END'
+#! /bin/sh
+echo "% test skipped %"
+exit 77
+END
+
+cat > xfail.test <<'END'
+#! /bin/sh
+echo "# expected failure #"
+exit 1
+END
+
+chmod a+x *.test
+
+test_log_edit ()
+{
+ sed -e "s|^ *$me 1\.0:.*$| $me 1.0: ???|" \
+ -e "s|^=====*|=======================|" $*
+}
+
+test_log_expected ()
+{
+ test_log_edit orig > exp
+ test_log_edit $1 > got
+ diff exp got || Exit 1
+ rm -f exp got
+}
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check
+ls -l
+cat test-suite.log
+cp test-suite.log orig
+
+$MAKE clean
+test -f test-suite.log && Exit 99 # Sanity check.
+
+# Check that we can override the testsuite log file at runtime.
+TEST_SUITE_LOG=zardoz.log $MAKE -e check
+ls -l
+test ! -f test-suite.log
+cat zardoz.log
+test_log_expected zardoz.log
+# Sanity check the distribution too (this also does minimal checks on
+# VPATH support).
+TEST_SUITE_LOG=zardoz.log $MAKE -e distcheck
+
+# Check that cleanup rules remove the correct file even when
+# user overrides are in place.
+cp orig test-suite.log
+TEST_SUITE_LOG=zardoz.log $MAKE -e clean
+ls -l
+test ! -f zardoz.log
+diff orig test-suite.log
+
+# Check that the default testsuite log doesn't get unduly modified.
+# Also check that the testsuite log file doesn't need to be named
+# accordingly to the `*.log' pattern.
+chmod a-w test-suite.log
+TEST_SUITE_LOG=TheLogFile $MAKE -e check
+ls -l
+diff orig test-suite.log
+test_log_expected TheLogFile
+TEST_SUITE_LOG=TheLogFile $MAKE -e clean
+ls -l
+test ! -f TheLogFile
+diff orig test-suite.log
+
+:
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: runtime redefinition of:
+# - $(TEST_SUITE_LOG) and $(TESTS)
+# - $(TEST_SUITE_LOG) and $(TEST_LOGS)
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = pass.test pass2.test skip.test skip2.test fail.test
+END
+
+cat > pass.test <<'END'
+#! /bin/sh
+exit 0
+END
+
+cp pass.test pass2.test
+
+cat > skip.test <<'END'
+#! /bin/sh
+echo "% skipped test %"
+exit 77
+END
+
+cp skip.test skip2.test
+
+cat > fail.test <<'END'
+#! /bin/sh
+exit 1
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+for test_list_override in \
+ 'TESTS=pass.test skip.test' \
+ 'TEST_LOGS=pass.log skip.log'
+do
+ env TEST_SUITE_LOG=partial.log "$test_list_override" \
+ $MAKE -e check >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ ls -l
+ cat pass.log
+ cat skip.log
+ cat partial.log
+ test ! -f test-suite.log
+ test ! -f pass2.log
+ test ! -f skip2.log
+ test ! -f fail.log
+ grep '^PASS: .*pass\.test' stdout
+ grep '^SKIP: .*skip\.test' stdout
+ $FGREP 'skip.test' partial.log
+ $FGREP '% skipped test %' partial.log
+ for t in pass2 skip2 fail; do
+ $FGREP "$t.test" stdout && Exit 1
+ $FGREP "$t.test" partial.log && Exit 1
+ done
+ rm -f *.log
+done
+
+:
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: runtime redefinition of $(TEST_SUITE_LOG)
+# for the recheck target.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = foo.test bar.test baz.test
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 0
+END
+
+cat > bar.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 99
+END
+
+cat > baz.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit ${BAZ_EXIT_STATUS-1}
+END
+
+chmod a+x *.test
+
+unset BAZ_EXIT_STATUS || :
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+chmod a-rw test-suite.log
+TEST_SUITE_LOG=my.log $MAKE -e recheck >stdout \
+ && { cat stdout; Exit 1; }
+cat stdout
+ls -l
+grep '^2 of 2 .*failed' stdout
+for x in stdout my.log; do
+ $FGREP foo.test $x && Exit 1
+ $FGREP bar.test $x
+ $FGREP baz.test $x
+done
+
+chmod a-rw my.log
+BAZ_EXIT_STATUS=0 TEST_SUITE_LOG=my2.log $MAKE -e recheck >stdout \
+ && { cat stdout; Exit 1; }
+cat stdout
+ls -l
+grep '^1 of 2 .*failed' stdout
+$FGREP foo.test stdout && Exit 1
+$FGREP bar.test stdout
+$FGREP baz.test stdout
+$FGREP foo.test my2.log && Exit 1
+$FGREP bar.test my2.log
+$FGREP baz.test my2.log && Exit 1
+
+chmod u+r test-suite.log my.log
+$FGREP baz.test test-suite.log
+$FGREP baz.test my.log
+
+:
projects / thirdparty / automake.git / commitdiff
