From: Stefano Lattarini Date: Sun, 22 Apr 2012 13:15:02 +0000 (+0200) Subject: [ng] coverage: parallel-tests and dynamic $(TESTS) content X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a9f1c30d11a84393953264c4444125916f24769b;p=thirdparty%2Fautomake.git [ng] coverage: parallel-tests and dynamic $(TESTS) content The recent changes to the parallel-tests implementation should allow us to define or override TESTS with more dynamic contents; in particular, something like: TESTS = $(wildcard t[0-9][0-9]*.sh) should now work smoothly. Let's cover this enhanced functionalities explicitly in our testsuite. * t/parallel-tests-dynamic.sh: New test. * t/list-of-tests.mk: Add it. * NG-NEWS: Document the new feature. Signed-off-by: Stefano Lattarini --- diff --git a/NG-NEWS b/NG-NEWS index 62c666af3..95b726223 100644 --- a/NG-NEWS +++ b/NG-NEWS @@ -79,10 +79,10 @@ Parallel testsuite harness * The new implementation of the parallel-tests harness tries to defer as much as possible processing on the $(TESTS) variable at make, rather than doing it at automake time. This as some positive effects - (FIXME: describe and quantify them), but also entails a minor annoyance. - When automake runs, it cannot know anymore whether $(TESTS) contains - or not any entry with no recognized test suffix; as a consequence, it - always need the '$(LOG_DRIVER)' variable to be defined. If it's not, + (described below), but also entails a minor annoyance. When automake + runs, it cannot know anymore whether $(TESTS) contains or not any + entry with no recognized test suffix; as a consequence, it always + need the '$(LOG_DRIVER)' variable to be defined. If it's not, automake will define it to a proper invocation of the 'test-driver' auxiliary script, and require that script to be present in the config-aux directory (or bring it in if "--add-missing" is in use). @@ -92,6 +92,12 @@ Parallel testsuite harness configure.ac: AC_SUBST([LOG_DRIVER], ['unused but required by automake']) +* The contents of $(TESTS) are mostly analyzed at make runtime rather + than at automake runtime, so that it is now possible to use GNU make + advanced features (like specific builtins or user-defined functions) + in the definition of $(TESTS); for example, the following should now + work as expected: + TESTS = $(wildcard $(srcdir)/t[0-9][0-9]*.sh) Miscellaneous ============= diff --git a/t/list-of-tests.mk b/t/list-of-tests.mk index 7d12d0f89..e3febf191 100644 --- a/t/list-of-tests.mk +++ b/t/list-of-tests.mk @@ -772,6 +772,7 @@ t/parallel-tests-console-output.sh \ t/parallel-tests-once.sh \ t/parallel-tests-trailing-bslash.sh \ t/parallel-tests-many.sh \ +t/parallel-tests-dynamic.sh \ t/tests-environment.sh \ t/am-tests-environment.sh \ t/tests-environment-backcompat.sh \ diff --git a/t/parallel-tests-dynamic.sh b/t/parallel-tests-dynamic.sh new file mode 100755 index 000000000..197046af2 --- /dev/null +++ b/t/parallel-tests-dynamic.sh @@ -0,0 +1,146 @@ +#! /bin/sh +# Copyright (C) 2012 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check that dynamic content for $(TESTS) and $(TEST_LOGS) is +# supported. + +am_parallel_tests=yes +. ./defs || Exit 1 + +echo AC_OUTPUT >> configure.ac + +cat > ok <<'END' +#!/bin/sh +exit 0 +END + +cat > ko <<'END' +#!/bin/sh +exit 1 +END + +cat > er << 'END' +#!/bin/sh +echo $0 should not be run >&2 +exit 99 +END + +chmod a+x ko ok + +mkdir t +cp ok t/nosuffix + +cp ok g1.sh +cp ok g2.sh +cp ok g3.sh +cp ok g4.sh +cp er g5.sh + +cp ok t00-foo.sh +cp ok t02.sh +cp ok t57_mu.sh +cp ok t7311.sh +cp ko t98S.sh +cp ko t99.sh +cp er t1.sh +cp er t9.sh +cp er tx98.sh + +cat > get-tests-list < Makefile.am << 'END' +my_add_dirprefix = $(strip $(1))/$(strip $(2)) +EXTRA_DIST = $(TESTS) get-tests-list +TEST_EXTENSIONS = .sh +TESTS = $(wildcard t[0-9][0-9]*.sh) $(shell $(srcdir)/get-tests-list) +TESTS += $(call my_add_dirprefix, t, nosuffix) +XFAIL_TESTS = $(wildcard t9[0-9]*.sh) +END + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +./configure + +$MAKE check > stdout || { cat stdout; Exit 1; } +cat stdout + +count_test_results total=11 pass=9 fail=0 xpass=0 xfail=2 skip=0 error=0 + +grep '^PASS: t/nosuffix$' stdout +grep '^PASS: g1\.sh$' stdout +grep '^PASS: g2\.sh$' stdout +grep '^PASS: g3\.sh$' stdout +grep '^PASS: g4\.sh$' stdout +grep '^PASS: t00-foo\.sh' stdout +grep '^PASS: t02\.sh' stdout +grep '^PASS: t57_mu\.sh' stdout +grep '^PASS: t7311\.sh' stdout +grep '^XFAIL: t98S\.sh' stdout +grep '^XFAIL: t99\.sh' stdout + +$MAKE mostlyclean +test "`find . -name *.log`" = ./config.log + +$MAKE distcheck > stdout || { cat stdout; Exit 1; } +cat stdout +# FIXME: this is currently broken, as the $(wildcard ...) call miss the +# FIXME: $(srcdir) component ... +#count_test_results total=11 pass=9 fail=0 xpass=0 xfail=2 skip=0 error=0 + +$MAKE check tests1='$(wildcard t00*.sh t98?.sh)' \ + tests2='$(shell ./get-tests-list | sed 1d)' \ + TESTS='$(tests1) $(tests2)' \ + > stdout || { cat stdout; Exit 1; } +cat stdout + +count_test_results total=4 pass=3 fail=0 xpass=0 xfail=1 skip=0 error=0 + +grep '^PASS: g3\.sh$' stdout +grep '^PASS: g4\.sh$' stdout +grep '^PASS: t00-foo\.sh' stdout +grep '^XFAIL: t98S\.sh' stdout + +$MAKE mostlyclean +test "`find . -name *.log`" = ./config.log + +# We can also override $(TEST_LOGS) dynamically. +$MAKE check TEST_LOGS='$(shell echo t00 | sed "s/$$/-foo.log/") t99.log' +test -f t00-foo.log +test -f t99.log + +# A little tricky in that we rely on the .log files created by +# the previous run to be present. +$MAKE check TEST_LOGS="\ + \$(wildcard t[0-9]*.log) \ + \$(call my_add_dirprefix, t, nosuffix).log \ +" > stdout || { cat stdout; Exit 1; } +cat stdout + +count_test_results total=3 pass=2 fail=0 xpass=0 xfail=1 skip=0 error=0 +grep '^PASS: t00-foo\.sh' stdout +grep '^PASS: t/nosuffix' stdout +grep '^XFAIL: t99\.sh' stdout + +: