Author: Alex Rousskov <rousskov@measurement-factory.com>

TestBed: Support multiple test spec arguments.

If at least one test fails, the script exits with a non-zero code (but
possibly not immediately, see --keep-going).

Each test spec is a test config file name or a well-known config name
(no path or extension!). If no specs are given, all known test specs are
used (as before). The same happens if the only test spec given is 'all'.
The following are now equivalent:

    ./test-builds.sh
    ./test-builds.sh all
    ./test-builds.sh btlayer-00-default btlayer-01-minimal btlayer-02-maximus
    ./test-builds.sh test-suite/buildtests/layer-*

You can mix file names and spec names, but not the 'all' macro: There is
currently no support for using 'all' together with other test cases.

Tolerate individual test errors if --keep-going is specified. This helps
when one wants to find more errors than just the first one, especially
when tests are long and are running without a human watching.

When detecting test failures, rely on test-suite/buildtest.sh exit
status code rather than on the presence of error-like strings in the log
file.

diff --git a/test-builds.sh b/test-builds.sh
index b53080d892792e48b25e954492f8ef176db51ad1..96ce1721ad61f8bd7e159c72dc72adaf49f14a63 100755 (executable)
--- a/test-builds.sh
+++ b/test-builds.sh
@@ -1,11 +1,14 @@
 #!/bin/sh
 #
-#  Run specific build tests for a given OS environment.
+#  Run all or some build tests for a given OS environment.
 #
 top=`dirname $0`
 
+globalResult=0
+
 cleanup="no"
 verbose="no"
+keepGoing="no"
 while [ $# -ge 1 ]; do
     case "$1" in
     --cleanup)
@@ -16,15 +19,16 @@ while [ $# -ge 1 ]; do
        verbose="yes"
        shift
        ;;
+    --keep-going)
+       keepGoing="yes"
+       shift
+       ;;
     *)
        break
        ;;
     esac
 done
 
-# Things to catch
-errors="^ERROR|\ error:|\ Error\ |No\ such|assertion\ failed|FAIL:"
-
 logtee() {
     if [ $verbose = yes ]; then
        tee $1
@@ -35,58 +39,93 @@ logtee() {
 
 buildtest() {
     opts=$1
-    layer=`basename $opts .opts`
-    btlayer="bt$layer"
+    layer=`basename ${opts} .opts`
+    btlayer="bt${layer}"
     log=${btlayer}.log
     echo "TESTING: ${layer}"
     rm -f -r ${btlayer} && mkdir ${btlayer}
     {
+       result=255
        cd ${btlayer}
        if test -e $top/test-suite/buildtest.sh ; then
-               $top/test-suite/buildtest.sh $opts
+           $top/test-suite/buildtest.sh ${opts} 2>&1
+           result=$?
        elif test -e ../$top/test-suite/buildtest.sh ; then
-               ../$top/test-suite/buildtest.sh ../$opts
+           ../$top/test-suite/buildtest.sh ../${opts} 2>&1
+           result=$?
+       else
+           echo "Error: cannot find $top/test-suite/buildtest.sh script"
+           result=1
        fi
-    } 2>&1 | logtee $log
+
+       # log the result for the outer script to notice
+       echo "buildtest.sh result is $result";
+    } 2>&1 | logtee ${log}
+
+    result=1 # failure by default
+    if grep -q '^buildtest.sh result is 0$' ${log}; then
+       result=0
+    fi
+
+    # Display BUILD parameters to double check that we are building the
+    # with the right parameters. TODO: Make less noisy.
     grep -E "BUILD" ${log}
-    grep -E "${errors}" $log && exit 1
+
+    errors="^ERROR|\ error:|\ Error\ |No\ such|assertion\ failed|FAIL:"
+    grep -E "${errors}" ${log}
+
     if test "${cleanup}" = "yes" ; then
        echo "REMOVE DATA: ${btlayer}"
        rm -f -r ${btlayer}
     fi
-    result=`tail -2 $log | head -1`
-    if test "${result}" = "Build Successful." ; then
-        echo "${result}"
+
+    if test $result -eq 0; then
+       # successful execution
+       if test "$verbose" = yes; then
+           echo "Build OK. Global result is $globalResult."
+       fi
     else
-        echo "Build Failed:"
-        tail -5 $log
-        exit 1
+        echo "Build Failed. Last log lines are:"
+        tail -5 ${log}
+       globalResult=1
     fi
+
     if test "${cleanup}" = "yes" ; then
        echo "REMOVE LOG: ${log}"
-       rm -f -r $log
+       rm -f -r ${log}
     fi
 }
 
-# Run a single test build by name or opts file
-if [ -e "$1" ]; then 
-
-       buildtest $1
-       exit 0
-fi
-tmp=`basename "${1}" .opts`
-if test -e $top/test-suite/buildtests/${tmp}.opts ; then
-       buildtest $top/test-suite/buildtests/${tmp}.opts
-       exit 0
+# Decide what tests to run, $* contains test spec names or filenames.
+# Use all knows specs if $* is empty or a special macro called 'all'.
+if test -n "$*" -a "$*" != all; then
+    tests="$*"
+else
+    tests=`ls -1 $top/test-suite/buildtests/layer*.opts`
 fi
 
-#
-#  Run specific tests for each combination of configure-time
-#  Options.
-#
-#  These layers are constructed from detailed knowledge of
-#  component dependencies.
-#
-for f in `ls -1 $top/test-suite/buildtests/layer*.opts` ; do
-       buildtest $f
+for t in $tests; do
+    if test -e "$t"; then 
+       # A configuration file
+        cfg="$t"
+    elif test -e "$top/test-suite/buildtests/${t}.opts"; then
+       # A well-known configuration name
+       cfg="$top/test-suite/buildtests/${t}.opts"
+    else
+       echo "Error: Unknown test specs '$t'"
+       cfg=''
+       globalResult=1
+    fi
+
+    # run the test, if any
+    if test -n "$cfg"; then
+       buildtest $cfg
+    fi
+
+    # quit on errors unless we should $keepGoing
+    if test $globalResult -ne 0 -a $keepGoing != yes; then
+       exit $globalResult
+    fi
 done
+
+exit $globalResult

diff --git a/test-suite/buildtest.sh b/test-suite/buildtest.sh
index 8c813562b88c4b5aba3d9c8f22b3c7f92189c82e..873dbf85b22a428f4c76564903cb1f3b8f6a1903 100755 (executable)
--- a/test-suite/buildtest.sh
+++ b/test-suite/buildtest.sh
@@ -36,9 +36,13 @@ fi
 #
 # above command currently encounters dependancy problems on cleanup.
 #
+# do not build any of the install's ...
 rm -f -r src/fs/aufs/.deps src/fs/diskd/.deps &&
        $base/../configure --silent ${OPTS} 2>&1 &&
        make ${pjobs} check 2>&1 &&
        make ${pjobs} 2>&1
 
-# do not build any of the install's ...
+# Remember and then explicitly return the result of the last command
+# to the script caller. Probably not needed on most or all platforms.
+result=$?
+exit ${result}

diff --git a/test-suite/buildtests/layer-02-maximus.opts b/test-suite/buildtests/layer-02-maximus.opts
index 5043210fdcc74b5185ee5a0fa10723f976cacc26..559ad6a2ec8a3a1cd815527cf2f8c7cb5c23ceb7 100644 (file)
--- a/test-suite/buildtests/layer-02-maximus.opts
+++ b/test-suite/buildtests/layer-02-maximus.opts
@@ -1,7 +1,7 @@
 #
-# Minimal configuration options.
-# - Everthing that can be disabled is
-# - Everyhing that can be done without is 
+# All configuration options that can be enabled are enabled,
+# XXX: with the exception of those that depend on the environment.
+# TODO: Add environment-specific tests to enable more options.
 #
 # The options for this level can be easily generated semi-automatically from configure.in by:
 #      grep -E "^AC_ARG_ENABLE" ./configure.in | grep -o -E "[0-9a-z\-]+[,]" | grep -o -E "[^,]+" >disable.opts
@@ -24,7 +24,8 @@
 #   --with-tags=TAGS \
 #
 #      Following features require special support from other optional packages.
-#      We can't test them automatically everywhere
+#      We can't test them automatically everywhere without detecting those
+#      optional packages first.
 #
 #   --enable-ecap \
 #   --enable-epoll \