+2011-07-31 Stefano Lattarini <stefano.lattarini@gmail.com>
+
+ simple tests: support developer-defined fd redirections
+ Motivated by coreutils bug#8846, and related discussions:
+ <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=8846>
+ <http://lists.gnu.org/archive/html/bug-autoconf/2011-06/msg00002.html>
+ In those threads it has been shown how problematic it can be to do
+ portable file descriptor redirections in the testsuite when using
+ the Automake testsuite harness. This change should remedy to that
+ situation.
+ * lib/am/check2.am (?GENERIC?%EXT%$(EXEEXT).log,
+ ?GENERIC?%EXT%.log, ?!GENERIC?%OBJ%): Append redirection defined
+ in $(AM_TESTS_FD_REDIRECT) to the command-line invocations of the
+ test scripts.
+ * lib/am/check.am [!%?PARALLEL_TESTS%] $(check-TESTS): Likewise.
+ * NEWS: Update.
+ * doc/automake.texi (Script-based Testsuites): Document the new
+ feature.
+ * tests/check-fd-redirect.test: New test.
+ * tests/parallel-tests-fd-redirect.test: Likewise.
+ * tests/parallel-tests-am_tests_environment.test: Remove checks
+ about the use of redirections in AM_TESTS_ENVIRONMENT: they would
+ check deprecated (if not undefined) behaviour now. Strengthen a
+ couple of still valid checks, to keep the test more in sync with
+ the documentation. Improve debugging information.
+ * tests/Makefile.am (TESTS): Update.
+
2011-07-22 Stefano Lattarini <stefano.lattarini@gmail.com>
docs: rework and extend documentation on testsuites support
Consistently with the existing syntax, this can be done by defining
special makefile variables `LOG_DRIVER' and `<ext>_LOG_DRIVER'.
+ - A new developer-reserved variable `AM_TESTS_FD_REDIRECT' can be used
+ to redirect/define file descriptors used by the test scripts.
+
* WARNING: Future backward-incompatibilities!
- The Automake support for automatic de-ANSI-fication will be removed in
former; for this to work portably, however, the contents of a non-empty
@code{AM_TESTS_ENVIRONMENT} @emph{must} be terminated by a semicolon.
+@vindex AM_TESTS_FD_REDIRECT
+The @code{AM_TESTS_FD_REDIRECT} variable can be used to define file
+descriptor redirections for the test scripts. One might think that
+@code{AM_TESTS_ENVIRONMENT} could be used for this purpose, but experience
+has shown that doing so portably is practically impossible. The main
+hurdle is constituted by Korn shells, which usually set the close-on-exec
+flag on file descriptors opened with the @command{exec} builtin, thus
+rendering an idiom like @code{AM_TESTS_ENVIRONMENT = exec 9>&2;}
+ineffectual. This issue also affects some Bourne shells, such as the
+HP-UX's @command{/bin/sh},
+@c FIXME: should we offer a link to the relevant discussions on the
+@c bug-autoconf list?
+
@example
AM_TESTS_ENVIRONMENT = \
## Some environment initializations are kept in a separate shell file
PATH=/usr/xpg4/bin:$$PATH; export PATH; \
fi;
@c $$ restore font-lock
+## With this, the test scripts will be able to print diagnostic messages
+## to the original standard error stream, even if the test driver
+## redirects the stderr of the test scripts to a log file before executing
+## them.
+AM_TESTS_FD_REDIRECT = 9>&2
@end example
@noindent
use of @code{VERBOSE} environment variable to get verbose output on
testsuite failures;
@item
-definition and honoring of @code{TESTS_ENVIRONMENT} and
-@code{AM_TESTS_ENVIRONMENT} variables;
+definition and honoring of @code{TESTS_ENVIRONMENT},
+@code{AM_TESTS_ENVIRONMENT} and @code{AM_TESTS_FD_REDIRECT}
+variables;
@item
definition of generic and extension-specific @code{LOG_COMPILER} and
@code{LOG_FLAGS} variables.
.pl.log:
@p='$<'; $(am__check_pre) \
$(PL_LOG_DRIVER) $(am__test_driver_flags) $(AM_PL_LOG_DRIVER_FLAGS) $(PL_LOG_DRIVER_FLAGS) -- \
- $(PL_LOG_COMPILE) "$$tst"
+ $(PL_LOG_COMPILE) "$$tst" $(AM_TESTS_FD_REDIRECT)
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
## why we also try `dir='
elif test -f $$tst; then dir=; \
else dir="$(srcdir)/"; fi; \
- if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
+ if $(TESTS_ENVIRONMENT) $${dir}$$tst $(AM_TESTS_FD_REDIRECT); then \
## Success
all=`expr $$all + 1`; \
case " $(XFAIL_TESTS) " in \
?!GENERIC?%OBJ%: %SOURCE%
@p='%SOURCE%'; $(am__check_pre) \
%DRIVER% $(am__test_driver_flags) %DRIVER_FLAGS% -- \
- %COMPILE% "$$tst"
+ %COMPILE% "$$tst" $(AM_TESTS_FD_REDIRECT)
## If no programs are built in this package, then this rule is removed
## at automake time. Otherwise, %am__EXEEXT% expands to a configure time
?GENERIC?%EXT%$(EXEEXT).log:
@p='%SOURCE%'; $(am__check_pre) \
%DRIVER% $(am__test_driver_flags) %DRIVER_FLAGS% -- \
- %COMPILE% "$$tst"
+ %COMPILE% "$$tst" $(AM_TESTS_FD_REDIRECT)
endif %am__EXEEXT%
check11.test \
check12.test \
check-exported-srcdir.test \
+check-fd-redirect.test \
check-tests-in-builddir.test \
check-tests_environment.test \
check-no-test-driver.test \
parallel-tests8.test \
parallel-tests9.test \
parallel-tests10.test \
+parallel-tests-fd-redirect.test \
parallel-tests-am_tests_environment.test \
parallel-tests-unreadable-log.test \
parallel-tests-subdir.test \
tap-plan-corner2.test tap-message-0.test tap-signal.test \
txinfo5.test $(instspc_xfail_tests)
parallel_tests = backcompat5-p.test check-exported-srcdir-p.test \
- check-tests-in-builddir-p.test check-tests_environment-p.test \
- check-p.test check11-p.test check12-p.test check2-p.test \
- check3-p.test check4-p.test check5-p.test check6-p.test \
- check7-p.test check8-p.test check9-p.test color-p.test \
- color2-p.test comment9-p.test dejagnu-p.test exeext4-p.test \
- maken3-p.test maken4-p.test posixsubst-tests-p.test
+ check-fd-redirect-p.test check-tests-in-builddir-p.test \
+ check-tests_environment-p.test check-p.test check11-p.test \
+ check12-p.test check2-p.test check3-p.test check4-p.test \
+ check5-p.test check6-p.test check7-p.test check8-p.test \
+ check9-p.test color-p.test color2-p.test comment9-p.test \
+ dejagnu-p.test exeext4-p.test maken3-p.test maken4-p.test \
+ posixsubst-tests-p.test
instspc_tests = instspc-squote-build.test instspc-squote-install.test \
instspc-dquote-build.test instspc-dquote-install.test \
instspc-bquote-build.test instspc-bquote-install.test \
check11.test \
check12.test \
check-exported-srcdir.test \
+check-fd-redirect.test \
check-tests-in-builddir.test \
check-tests_environment.test \
check-no-test-driver.test \
parallel-tests8.test \
parallel-tests9.test \
parallel-tests10.test \
+parallel-tests-fd-redirect.test \
parallel-tests-am_tests_environment.test \
parallel-tests-unreadable-log.test \
parallel-tests-subdir.test \
.test.log:
@p='$<'; $(am__check_pre) \
$(TEST_LOG_DRIVER) $(am__test_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS) $(TEST_LOG_DRIVER_FLAGS) -- \
- $(TEST_LOG_COMPILE) "$$tst"
+ $(TEST_LOG_COMPILE) "$$tst" $(AM_TESTS_FD_REDIRECT)
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
backcompat5-p.log: backcompat5.test
check-exported-srcdir-p.log: check-exported-srcdir.test
+check-fd-redirect-p.log: check-fd-redirect.test
check-tests-in-builddir-p.log: check-tests-in-builddir.test
check-tests_environment-p.log: check-tests_environment.test
check-p.log: check.test
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Simple Tests support: redirection of file descriptors with
+# AM_TESTS_FD_REDIRECT.
+# See also related test 'parallel-tests-fd-redirect.test'.
+
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = foo.test
+AM_TESTS_FD_REDIRECT = 3<three 4>four 5>>five 7<&0 8>&1 9>&2
+END
+
+echo '3333' > three
+chmod a-w three
+
+: > foo.test
+chmod a+x foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+./configure
+
+do_check ()
+{
+ cat foo.test # For debugging.
+ echo 'this line will be removed' > four
+ echo 'this line will not be removed' > five
+ st=0
+ echo 'ok ok ok' | $MAKE check >stdout 2>stderr || st=1
+ cat stdout
+ cat stderr >&2
+ cat four
+ test x"$parallel_tests" != x"yes" || cat foo.log
+ test $st -eq 0
+ grep '[ /]foo\.test: foofoofoo$' stdout
+ grep '[ /]foo\.test: barbarbar$' stderr
+ grep 'this line' four && Exit 1
+ grep '^3333$' four
+ grep '^this line will not be removed$' five
+ grep '^ok ok ok$' five
+ $EGREP '(foofoofoo|barbarbar|3333|ok ok ok|this line)' foo.log && Exit 1
+ :
+}
+
+# Try using both shell script and a perl script as the test, for
+# better coverage.
+
+cat > foo.test <<'END'
+#! /bin/sh
+set -e
+
+read FOO <&3
+test 3333 -eq "$FOO"
+echo "$FOO" >&4
+
+grep '^ok ok ok$' <&7 >&5
+
+echo " " $0: foofoofoo >&8
+echo " " $0: barbarbar >&9
+END
+
+do_check
+
+echo "#! $PERL -w" > foo.test
+cat >> foo.test <<'END'
+use warnings FATAL => 'all';
+use strict;
+
+open (FD3, "<&=3") or die "opening FD3: $!";
+open (FD4, ">&=4") or die "opening FD4: $!";
+open (FD5, ">&=5") or die "opening FD5: $!";
+open (FD7, "<&=7") or die "opening FD7: $!";
+open (FD8, ">&=8") or die "opening FD8: $!";
+open (FD9, ">&=9") or die "opening FD9: $!";
+
+chomp (my $FOO = <FD3>);
+die "$FOO != 3333" if not $FOO eq "3333";
+print FD4 "$FOO\n";
+
+chomp ($_ = <FD7>);
+die "$_ != 'ok ok ok'" if not $_ eq 'ok ok ok';
+print FD5 "$_\n";
+
+print FD8 " $0: foofoofoo\n";
+print FD9 " $0: barbarbar\n";
+END
+
+do_check
+
+:
TEST_EXTENSIONS = .sh .test
TESTS = foo.test bar.sh
SH_LOG_COMPILER = sh
-AM_TESTS_ENVIRONMENT = $(MKDIR_P) quux.dir; exec 9>&2; fd=9; export fd;
-EXTRA_DIST = $(TESTS)
+AM_TESTS_ENVIRONMENT = \
+ $(MKDIR_P) quux.dir; \
+ if test -f $(srcdir)/test-env.sh; then \
+ . $(srcdir)/test-env.sh; \
+ fi; \
+ FOO=1; export FOO;
+EXTRA_DIST = $(TESTS) test-env.sh
END
cat > foo.test << 'END'
#! /bin/sh
-echo "FOO='$FOO'"
-echo "BAR='$BAR'"
-test x"$FOO" = x"$BAR"
+ls -l && test -d quux.dir
END
chmod a+x foo.test
cat > bar.sh << 'END'
-: ${fd=2}
-eval "echo 'diagnostic message from test bar' >&${fd}"
+echo "FOO='$FOO'"
+echo "BAR='$BAR'"
+test x"$FOO" = x"$BAR"
END
-minicheck ()
+debug_info ()
{
+ cat test-suite.log
cat foo.log
cat bar.log
+}
+
+minicheck ()
+{
+ debug_info
test -d quux.dir
}
./configure
-FOO=1; export FOO
-
-TESTS_ENVIRONMENT='BAR=1' $MAKE check || { cat test-suite.log; Exit 1; }
-minicheck
-miniclean
-TESTS_ENVIRONMENT='BAR=2' $MAKE check && { cat test-suite.log; Exit 1; }
+TESTS_ENVIRONMENT='BAR=1' $MAKE check || { debug_info; Exit 1; }
minicheck
miniclean
-BAR=$FOO; export BAR
-
-$MAKE check 2>stderr || { cat stderr >&2; Exit1; }
-cat stderr >&2
+TESTS_ENVIRONMENT='BAR=2' $MAKE check && { debug_info; Exit 1; }
minicheck
-grep 'diagnostic message from test bar' stderr
-grep 'diagnostic message from test bar' bar.log && Exit 1
miniclean
-TESTS_ENVIRONMENT='fd=2' $MAKE check 2>stderr || { cat stderr >&2; Exit1; }
-cat stderr >&2
+echo 'BAR=1 && export BAR' > test-env.sh
+$MAKE check || { debug_info; Exit 1; }
minicheck
-grep 'diagnostic message from test bar' stderr && Exit 1
-grep 'diagnostic message from test bar' bar.log
-miniclean
-
$MAKE distcheck
:
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests support: redirection of file descriptors with
+# AM_TESTS_FD_REDIRECT, even when using tests without suffix, or
+# which are binary executables.
+# We use some tricks to ensure that all code paths in `lib/am/check2.am'
+# are covered, even on platforms where $(EXEEXT) would be naturally empty.
+# See also the more generic test 'check-fd-redirect.test'.
+
+required='cc native'
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_PROG_CC
+AM_CONDITIONAL([real_EXEEXT], [test -n "$EXEEXT"])
+test -n "$EXEEXT" || EXEEXT=.bin
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+AM_TESTS_FD_REDIRECT = 9>&1
+TEST_EXTENSIONS = .test .sh
+TESTS = foo.sh bar $(check_PROGRAMS)
+check_PROGRAMS = baz qux.test
+qux_test_SOURCES = zardoz.c
+
+## Sanity check.
+if !real_EXEEXT
+check-local:
+ test -f baz.bin
+ test -f qux.test.bin
+endif
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+cat > foo.sh <<'END'
+#!/bin/sh
+echo " foofoofoo" >&9
+END
+chmod a+x foo.sh
+
+cat > bar <<'END'
+#!/bin/sh
+echo " barbarbar" >&9
+END
+
+chmod a+x foo.sh bar
+
+cat > baz.c <<'END'
+#include <unistd.h>
+int main (void)
+{
+ write (9, " bazbazbaz\n", 11);
+ return 0;
+}
+END
+
+cat > zardoz.c <<'END'
+#include <unistd.h>
+int main (void)
+{
+ write (9, " quxquxqux\n", 11);
+ return 0;
+}
+END
+
+./configure
+
+# Sanity checks.
+st=0
+grep '^bar\.log:.*bar' Makefile || st=1
+grep '^baz\.log:.*baz\$(EXEEXT)' Makefile || st=1
+grep '^\.test\$(EXEEXT)\.log:' Makefile || st=1
+$EGREP '^(foo|qux)\.log:' Makefile && st=1
+test $st -eq 0 || fatal_ "doesn't cover expected code paths"
+
+st=0
+$MAKE check >stdout || st=1
+cat stdout
+cat foo.log
+cat bar.log
+cat baz.log
+cat qux.log
+test $st -eq 0
+grep "^ foofoofoo$" stdout
+grep "^ barbarbar$" stdout
+grep "^ bazbazbaz$" stdout
+grep "^ quxquxqux$" stdout
+$EGREP '(foofoofoo|barbarbar|bazbazbaz|quxquxqux)' *.log && Exit 1
+
+:
projects / thirdparty / automake.git / commitdiff
