From 4abfd78bbf7b79282bc154f049d392104715086d Mon Sep 17 00:00:00 2001 From: Amos Jeffries Date: Mon, 23 Mar 2009 23:32:33 +1300 Subject: [PATCH] Author: Alex Rousskov TestBed: Support multiple test spec arguments. If at least one test fails, the script exits with a non-zero code (but possibly not immediately, see --keep-going). Each test spec is a test config file name or a well-known config name (no path or extension!). If no specs are given, all known test specs are used (as before). The same happens if the only test spec given is 'all'. The following are now equivalent: ./test-builds.sh ./test-builds.sh all ./test-builds.sh btlayer-00-default btlayer-01-minimal btlayer-02-maximus ./test-builds.sh test-suite/buildtests/layer-* You can mix file names and spec names, but not the 'all' macro: There is currently no support for using 'all' together with other test cases. Tolerate individual test errors if --keep-going is specified. This helps when one wants to find more errors than just the first one, especially when tests are long and are running without a human watching. When detecting test failures, rely on test-suite/buildtest.sh exit status code rather than on the presence of error-like strings in the log file. --- test-builds.sh | 111 +++++++++++++------- test-suite/buildtest.sh | 6 +- test-suite/buildtests/layer-02-maximus.opts | 9 +- 3 files changed, 85 insertions(+), 41 deletions(-) diff --git a/test-builds.sh b/test-builds.sh index b53080d892..96ce1721ad 100755 --- a/test-builds.sh +++ b/test-builds.sh @@ -1,11 +1,14 @@ #!/bin/sh # -# Run specific build tests for a given OS environment. +# Run all or some build tests for a given OS environment. # top=`dirname $0` +globalResult=0 + cleanup="no" verbose="no" +keepGoing="no" while [ $# -ge 1 ]; do case "$1" in --cleanup) @@ -16,15 +19,16 @@ while [ $# -ge 1 ]; do verbose="yes" shift ;; + --keep-going) + keepGoing="yes" + shift + ;; *) break ;; esac done -# Things to catch -errors="^ERROR|\ error:|\ Error\ |No\ such|assertion\ failed|FAIL:" - logtee() { if [ $verbose = yes ]; then tee $1 @@ -35,58 +39,93 @@ logtee() { buildtest() { opts=$1 - layer=`basename $opts .opts` - btlayer="bt$layer" + layer=`basename ${opts} .opts` + btlayer="bt${layer}" log=${btlayer}.log echo "TESTING: ${layer}" rm -f -r ${btlayer} && mkdir ${btlayer} { + result=255 cd ${btlayer} if test -e $top/test-suite/buildtest.sh ; then - $top/test-suite/buildtest.sh $opts + $top/test-suite/buildtest.sh ${opts} 2>&1 + result=$? elif test -e ../$top/test-suite/buildtest.sh ; then - ../$top/test-suite/buildtest.sh ../$opts + ../$top/test-suite/buildtest.sh ../${opts} 2>&1 + result=$? + else + echo "Error: cannot find $top/test-suite/buildtest.sh script" + result=1 fi - } 2>&1 | logtee $log + + # log the result for the outer script to notice + echo "buildtest.sh result is $result"; + } 2>&1 | logtee ${log} + + result=1 # failure by default + if grep -q '^buildtest.sh result is 0$' ${log}; then + result=0 + fi + + # Display BUILD parameters to double check that we are building the + # with the right parameters. TODO: Make less noisy. grep -E "BUILD" ${log} - grep -E "${errors}" $log && exit 1 + + errors="^ERROR|\ error:|\ Error\ |No\ such|assertion\ failed|FAIL:" + grep -E "${errors}" ${log} + if test "${cleanup}" = "yes" ; then echo "REMOVE DATA: ${btlayer}" rm -f -r ${btlayer} fi - result=`tail -2 $log | head -1` - if test "${result}" = "Build Successful." ; then - echo "${result}" + + if test $result -eq 0; then + # successful execution + if test "$verbose" = yes; then + echo "Build OK. Global result is $globalResult." + fi else - echo "Build Failed:" - tail -5 $log - exit 1 + echo "Build Failed. Last log lines are:" + tail -5 ${log} + globalResult=1 fi + if test "${cleanup}" = "yes" ; then echo "REMOVE LOG: ${log}" - rm -f -r $log + rm -f -r ${log} fi } -# Run a single test build by name or opts file -if [ -e "$1" ]; then - - buildtest $1 - exit 0 -fi -tmp=`basename "${1}" .opts` -if test -e $top/test-suite/buildtests/${tmp}.opts ; then - buildtest $top/test-suite/buildtests/${tmp}.opts - exit 0 +# Decide what tests to run, $* contains test spec names or filenames. +# Use all knows specs if $* is empty or a special macro called 'all'. +if test -n "$*" -a "$*" != all; then + tests="$*" +else + tests=`ls -1 $top/test-suite/buildtests/layer*.opts` fi -# -# Run specific tests for each combination of configure-time -# Options. -# -# These layers are constructed from detailed knowledge of -# component dependencies. -# -for f in `ls -1 $top/test-suite/buildtests/layer*.opts` ; do - buildtest $f +for t in $tests; do + if test -e "$t"; then + # A configuration file + cfg="$t" + elif test -e "$top/test-suite/buildtests/${t}.opts"; then + # A well-known configuration name + cfg="$top/test-suite/buildtests/${t}.opts" + else + echo "Error: Unknown test specs '$t'" + cfg='' + globalResult=1 + fi + + # run the test, if any + if test -n "$cfg"; then + buildtest $cfg + fi + + # quit on errors unless we should $keepGoing + if test $globalResult -ne 0 -a $keepGoing != yes; then + exit $globalResult + fi done + +exit $globalResult diff --git a/test-suite/buildtest.sh b/test-suite/buildtest.sh index 8c813562b8..873dbf85b2 100755 --- a/test-suite/buildtest.sh +++ b/test-suite/buildtest.sh @@ -36,9 +36,13 @@ fi # # above command currently encounters dependancy problems on cleanup. # +# do not build any of the install's ... rm -f -r src/fs/aufs/.deps src/fs/diskd/.deps && $base/../configure --silent ${OPTS} 2>&1 && make ${pjobs} check 2>&1 && make ${pjobs} 2>&1 -# do not build any of the install's ... +# Remember and then explicitly return the result of the last command +# to the script caller. Probably not needed on most or all platforms. +result=$? +exit ${result} diff --git a/test-suite/buildtests/layer-02-maximus.opts b/test-suite/buildtests/layer-02-maximus.opts index 5043210fdc..559ad6a2ec 100644 --- a/test-suite/buildtests/layer-02-maximus.opts +++ b/test-suite/buildtests/layer-02-maximus.opts @@ -1,7 +1,7 @@ # -# Minimal configuration options. -# - Everthing that can be disabled is -# - Everyhing that can be done without is +# All configuration options that can be enabled are enabled, +# XXX: with the exception of those that depend on the environment. +# TODO: Add environment-specific tests to enable more options. # # The options for this level can be easily generated semi-automatically from configure.in by: # grep -E "^AC_ARG_ENABLE" ./configure.in | grep -o -E "[0-9a-z\-]+[,]" | grep -o -E "[^,]+" >disable.opts @@ -24,7 +24,8 @@ # --with-tags=TAGS \ # # Following features require special support from other optional packages. -# We can't test them automatically everywhere +# We can't test them automatically everywhere without detecting those +# optional packages first. # # --enable-ecap \ # --enable-epoll \ -- 2.47.3