From: Stefano Lattarini <stefano.lattarini@gmail.com>
Date: Sat, 28 Jul 2012 09:54:27 +0000 (+0200)
Subject: [ng] tests: do not run very expensive tests by default
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b300a4cb19fc524d4d640f62754a2b803d8018ba;p=thirdparty%2Fautomake.git

[ng] tests: do not run very expensive tests by default

Some tests in the Automake testsuite (especially those testing for bugs
w.r.t. command-line length limits) require a *long* time to run.  For
example, the test 'parallel-tests-many.sh' (the most extreme example of
this issue) takes ~ 11 minutes to execute on a modern, blazingly fast
multi-core ppc64 system (64 cores at 3.5 GHz each).

This slow-down is unacceptable during routine runs of the testsuite,
that I do every bunch of commits (or every commit, for more tricky
changes).  And is even less acceptable for the casual user that just
run the testsuite before installing Automake, maybe on an aging and
slower system (or maybe on Cygwin *shudder*).

So, let's follow the route of GNU coreutils here: declare some tests as
"expensive", and let them run only if called if the RUN_EXPENSIVE_TESTS
environment variable is set to "yes".

* test-lib.sh (expensive_): New function; used in a test, cause it to be
skipped unless the RUN_EXPENSIVE_TESTS variable is defined to "yes".
* t/parallel-tests-many.sh, t/testsuite-summary-count-many.sh,
t/dist-many.sh: Call 'expensive_'.
* runtest.in: Export RUN_EXPENSIVE_TESTS to "yes": if a test is called
directly through './runtest', it should be run even it it's expensive.
* Makefile.am (check-expensive, installcheck-expensive): New convenience
targets that run the testsuite with RUN_EXPENSIVE_TESTS exported to
"yes", so that the expensive tests are not skipped.
* t/README: Update.

Signed-off-by: Stefano Lattarini <stefano.lattarini@gmail.com>
---

diff --git a/Makefile.am b/Makefile.am
index 362be992b..3ea096eec 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -492,6 +492,10 @@ installcheck-local: installcheck-testsuite
 installcheck-testsuite:
 	am_running_installcheck=yes $(MAKE) check
 
+# Run the testsuite without skipping expensive tests by default.
+check-expensive installcheck-expensive: %-expensive:
+	$(MAKE) $* RUN_EXPENSIVE_TESTS=yes
+
 # Performance tests.
 perf_TESTS = $(wildcard $(srcdir)/t/perf/*.sh) \
              $(wildcard $(srcdir)/t/perf/*.tap)
diff --git a/runtest.in b/runtest.in
index c2c5d1f9d..3d01e6823 100644
--- a/runtest.in
+++ b/runtest.in
@@ -27,6 +27,10 @@ set -e; set -u
 : ${abs_builddir='@abs_builddir@'}
 : ${PATH_SEPARATOR='@PATH_SEPARATOR@'}
 
+# If a test is called directly, it should be unconditionally run,
+# even it it's expensive.
+RUN_EXPENSIVE_TESTS=yes; export RUN_EXPENSIVE_TESTS
+
 # For sourcing of extra "shell libraries" by our test scripts.  As per
 # POSIX, sourcing a file with '.' will cause it to be looked up in $PATH
 # in case it is given with a relative name containing no slashes.
diff --git a/t/README b/t/README
index 570b66712..ee063b44c 100644
--- a/t/README
+++ b/t/README
@@ -8,10 +8,16 @@ User interface
 Running the tests
 -----------------
 
-  To run all tests:
+  To run (almost) all tests:
 
     make -k check
 
+  With the above, few tests (which are very time-consuming) will be skipped.
+  If you want to run them too, for extra coverage, you can use either of:
+
+    make -k check-expensive
+    make -k check RUN_EXPENSIVE_TESTS=yes
+
   By default, verbose output of a test 't/foo.sh' or 't/foo.tap' is retained
   in the log file 't/foo.log'.  Also, a summary log is created in the file
   'test-suite.log' (in the top-level directory).
diff --git a/t/ax/test-lib.sh b/t/ax/test-lib.sh
index 6048f8112..f6621d051 100644
--- a/t/ax/test-lib.sh
+++ b/t/ax/test-lib.sh
@@ -133,6 +133,13 @@ fi
 ##  Auxiliary shell functions.  ##
 ## ---------------------------- ##
 
+expensive_()
+{
+  if test x"$RUN_EXPENSIVE_TESTS" != x"yes"; then
+    skip_all_ 'expensive: disabled by default'
+  fi
+}
+
 # Tell whether we should keep the test directories around, even in
 # case of success.  By default, we don't.
 am_keeping_testdirs ()
diff --git a/t/dist-many.sh b/t/dist-many.sh
index 3fd0cb4fb..580d79ab2 100755
--- a/t/dist-many.sh
+++ b/t/dist-many.sh
@@ -21,6 +21,8 @@
 
 . ./defs || exit 1
 
+expensive_
+
 echo AC_OUTPUT >> configure.ac
 
 file=an-empty-file-with-a-long-name
diff --git a/t/parallel-tests-many.sh b/t/parallel-tests-many.sh
index bd107e204..70d9d2316 100755
--- a/t/parallel-tests-many.sh
+++ b/t/parallel-tests-many.sh
@@ -20,6 +20,8 @@
 
 . ./defs || exit 1
 
+expensive_
+
 echo AC_OUTPUT >> configure.ac
 
 cat > Makefile.am << 'END'
diff --git a/t/testsuite-summary-count-many.sh b/t/testsuite-summary-count-many.sh
index 938c91e2e..55b2eb42e 100755
--- a/t/testsuite-summary-count-many.sh
+++ b/t/testsuite-summary-count-many.sh
@@ -21,6 +21,8 @@
 
 . ./defs || exit 1
 
+expensive_
+
 for s in trivial-test-driver extract-testsuite-summary.pl; do
   cp "$am_testauxdir/$s" . || fatal_ "failed to fetch auxiliary script $s"
 done