[ng] tests: do not run very expensive tests by default

Some tests in the Automake testsuite (especially those testing for bugs
w.r.t. command-line length limits) require a *long* time to run.  For
example, the test 'parallel-tests-many.sh' (the most extreme example of
this issue) takes ~ 11 minutes to execute on a modern, blazingly fast
multi-core ppc64 system (64 cores at 3.5 GHz each).

This slow-down is unacceptable during routine runs of the testsuite,
that I do every bunch of commits (or every commit, for more tricky
changes).  And is even less acceptable for the casual user that just
run the testsuite before installing Automake, maybe on an aging and
slower system (or maybe on Cygwin *shudder*).

So, let's follow the route of GNU coreutils here: declare some tests as
"expensive", and let them run only if called if the RUN_EXPENSIVE_TESTS
environment variable is set to "yes".

* test-lib.sh (expensive_): New function; used in a test, cause it to be
skipped unless the RUN_EXPENSIVE_TESTS variable is defined to "yes".
* t/parallel-tests-many.sh, t/testsuite-summary-count-many.sh,
t/dist-many.sh: Call 'expensive_'.
* runtest.in: Export RUN_EXPENSIVE_TESTS to "yes": if a test is called
directly through './runtest', it should be run even it it's expensive.
* Makefile.am (check-expensive, installcheck-expensive): New convenience
targets that run the testsuite with RUN_EXPENSIVE_TESTS exported to
"yes", so that the expensive tests are not skipped.
* t/README: Update.

Signed-off-by: Stefano Lattarini <stefano.lattarini@gmail.com>

diff --git a/Makefile.am b/Makefile.am
index 362be992b2c7d6b56ea66b69cc52859da7c0d5c5..3ea096eec2674b664cf05937217849142f01a052 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -492,6 +492,10 @@ installcheck-local: installcheck-testsuite
 installcheck-testsuite:
        am_running_installcheck=yes $(MAKE) check
 
+# Run the testsuite without skipping expensive tests by default.
+check-expensive installcheck-expensive: %-expensive:
+       $(MAKE) $* RUN_EXPENSIVE_TESTS=yes
+
 # Performance tests.
 perf_TESTS = $(wildcard $(srcdir)/t/perf/*.sh) \
              $(wildcard $(srcdir)/t/perf/*.tap)

diff --git a/runtest.in b/runtest.in
index c2c5d1f9d7ca40d279008ee290826923f0fcaa71..3d01e68237fa7406e76d35d252dbede617ad7452 100644 (file)
--- a/runtest.in
+++ b/runtest.in
@@ -27,6 +27,10 @@ set -e; set -u
 : ${abs_builddir='@abs_builddir@'}
 : ${PATH_SEPARATOR='@PATH_SEPARATOR@'}
 
+# If a test is called directly, it should be unconditionally run,
+# even it it's expensive.
+RUN_EXPENSIVE_TESTS=yes; export RUN_EXPENSIVE_TESTS
+
 # For sourcing of extra "shell libraries" by our test scripts.  As per
 # POSIX, sourcing a file with '.' will cause it to be looked up in $PATH
 # in case it is given with a relative name containing no slashes.

diff --git a/t/README b/t/README
index 570b6671246ae79da732d31619b3597f4e080176..ee063b44c443dfca703206b0fdc34197e41b698d 100644 (file)
--- a/t/README
+++ b/t/README
@@ -8,10 +8,16 @@ User interface
 Running the tests
 -----------------
 
-  To run all tests:
+  To run (almost) all tests:
 
     make -k check
 
+  With the above, few tests (which are very time-consuming) will be skipped.
+  If you want to run them too, for extra coverage, you can use either of:
+
+    make -k check-expensive
+    make -k check RUN_EXPENSIVE_TESTS=yes
+
   By default, verbose output of a test 't/foo.sh' or 't/foo.tap' is retained
   in the log file 't/foo.log'.  Also, a summary log is created in the file
   'test-suite.log' (in the top-level directory).

diff --git a/t/ax/test-lib.sh b/t/ax/test-lib.sh
index 6048f8112d992bcb3cdab04f7aaccfc6949e1791..f6621d05161e15721ba300cdd8e8246f5d57cc86 100644 (file)
--- a/t/ax/test-lib.sh
+++ b/t/ax/test-lib.sh
@@ -133,6 +133,13 @@ fi
 ##  Auxiliary shell functions.  ##
 ## ---------------------------- ##
 
+expensive_()
+{
+  if test x"$RUN_EXPENSIVE_TESTS" != x"yes"; then
+    skip_all_ 'expensive: disabled by default'
+  fi
+}
+
 # Tell whether we should keep the test directories around, even in
 # case of success.  By default, we don't.
 am_keeping_testdirs ()

diff --git a/t/dist-many.sh b/t/dist-many.sh
index 3fd0cb4fb5d22d691c1c9a7e31e070c3dd945674..580d79ab2dc4e18328685a3ba91aaf292edcbfe1 100755 (executable)
--- a/t/dist-many.sh
+++ b/t/dist-many.sh
@@ -21,6 +21,8 @@
 
 . ./defs || exit 1
 
+expensive_
+
 echo AC_OUTPUT >> configure.ac
 
 file=an-empty-file-with-a-long-name

diff --git a/t/parallel-tests-many.sh b/t/parallel-tests-many.sh
index bd107e204b1a1897d548da867720fbc3bc1aeb95..70d9d2316882edfbe2cfe25795725ddd93499374 100755 (executable)
--- a/t/parallel-tests-many.sh
+++ b/t/parallel-tests-many.sh
@@ -20,6 +20,8 @@
 
 . ./defs || exit 1
 
+expensive_
+
 echo AC_OUTPUT >> configure.ac
 
 cat > Makefile.am << 'END'

diff --git a/t/testsuite-summary-count-many.sh b/t/testsuite-summary-count-many.sh
index 938c91e2ecd7406245fd7f32f46cde04efeb0004..55b2eb42e8e812c138c57ed3b030d121d4866fbd 100755 (executable)
--- a/t/testsuite-summary-count-many.sh
+++ b/t/testsuite-summary-count-many.sh
@@ -21,6 +21,8 @@
 
 . ./defs || exit 1
 
+expensive_
+
 for s in trivial-test-driver extract-testsuite-summary.pl; do
   cp "$am_testauxdir/$s" . || fatal_ "failed to fetch auxiliary script $s"
 done