[rt46602] Tidy up run management

Miscellaneous tidying up of run management.  The most significant
change is that "runall.sh" now runs _all_ the tests, even the
ones that can run in parallel.  runsequential.sh is the script
to run tests that have not been converted to parallel running.

(cherry picked from commit 32fe6f7682361225415cc6714a1e369dca94b636)
(cherry picked from commit a49afdaeb818fb81ead578ff21a8dcd2b89d00aa)
(cherry picked from commit dec6d02f2850b3c55c10fd4a0e0984e6d18593f7)

diff --git a/bin/tests/system/Makefile.in b/bin/tests/system/Makefile.in
index 58c32d19bae67c6b055dee2fc506a691c2767ae3..cf0c9d2fbab82c5517164a824861b786fddf8f2e 100644 (file)
--- a/bin/tests/system/Makefile.in
+++ b/bin/tests/system/Makefile.in
@@ -74,24 +74,19 @@ parallel.mk:
                echo "  @$(SHELL) ./run.sh -p $$port $$directory" >> $@ ; \
        done
 
-# Targets to run the tests that can be done in parallel (which have unique
-# ports assigned to them) and the tests which must be run sequentially (because
-# they all use query port 5300 and control port 9953).
+# Targets to run the tests.
 
-parallel: parallel.mk
-       @$(MAKE) -f parallel.mk
-
-sequential:
-       @if test -f ./runall.sh; then $(SHELL) ./runall.sh; fi
-
-# Standard targets.
+test: parallel.mk
+       @$(MAKE) -f parallel.mk check 2>&1 | tee systests.output
+       @$(SHELL) ./runsequential.sh 2>&1 | tee -a systests.output
+       @$(SHELL) ./testsummary.sh
 
 check: test
 
-test: parallel sequential
+# Other targets.
 
 testclean clean distclean::
-       if test -f ./cleanall.sh; then sh ./cleanall.sh; fi
+       if test -f ./cleanall.sh; then $(SHELL) ./cleanall.sh; fi
        rm -f systests.output
        rm -f random.data
        rm -f parallel.mk

diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index b19a6a62bf0560263f367868c4bccfce947396d1..1666b491dda083894f8552a2af77c9df01a4166c 100644 (file)
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -139,7 +139,7 @@ if test -t 1 && type tput > /dev/null 2>&1 ; then
     COLOR_FAIL=`tput setaf 1`   # red
     COLOR_WARN=`tput setaf 3`   # yellow
     COLOR_PASS=`tput setaf 2`   # green
-    COLOR_STEN=`tput setaf 4`   # blue
+    COLOR_STEN=`tput setaf 4`   # blue - for start/end messages
     COLOR_INFO=`tput bold`      # bold
     COLOR_NONE=`tput sgr0`
 else
@@ -188,10 +188,6 @@ fi
 TESTNAME="`basename $PWD`"
 
 echo_i() {
-    # Without formatting
-    echo "I:$TESTNAME:$@" >> test.output
-
-    # With formatting
     echoinfo "I:$TESTNAME:$@"
 }
 

diff --git a/bin/tests/system/getopts.sh b/bin/tests/system/getopts.sh
index b3ac264b63c12c4596a2841d6deb6acf49cdf594..001cad58d69aaf53bfde31aa91b25ba71188683b 100644 (file)
--- a/bin/tests/system/getopts.sh
+++ b/bin/tests/system/getopts.sh
@@ -42,7 +42,7 @@ OPTIND=1
 # the upper of the 10 ports notionally assigned does not exceed 65535.
 
 if [ "$((${port}+0))" != "${port}" ] || [ "${port}" -le 1024 ] || [ "${port}" -gt 65520 ]; then
-    echo "Specified port '$port' must be numeric and in the range 1025 to 65520" >&2
+    echo "Base of port range ($port) must be numeric and in the range 1025 to 65520" >&2
     exit 1
 fi
 
@@ -56,6 +56,11 @@ aport7=$(($port + 7))
 aport8=$(($port + 8))
 controlport=$(($port + 9))
 
+# Two more symbols that denote the limits of the range.
+
+portlow=$port
+porthigh=$controlport
+
 
 # copy_setports - Copy Configuration File and Replace Ports
 #

diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index 5043126911f17b72a4ca2fd8569edcda13173583..f3d62b9f424773adf69342d4cb4de86c57ee85cc 100644 (file)
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -52,8 +52,7 @@ test -d $test || { echofail "$0: $test: no such test" >&2; exit 1; }
 echoinfo "S:$test:`date $dateargs`" >&2
 echoinfo "T:$test:1:A" >&2
 echoinfo "A:$test:System test $test" >&2
-echoinfo "I:$test:PORT:${port}" >&2
-echoinfo "I:$test:CONTROLPORT:${controlport}" >&2
+echoinfo "I:$test:PORTRANGE:${portlow} - ${porthigh}"
 
 if [ x${PERL:+set} = x ]
 then
@@ -122,7 +121,7 @@ $PERL stop.pl $test
 status=`expr $status + $?`
 
 if [ $status != 0 ]; then
-       echofail "R:$test:$FAIL"
+       echofail "R:$test:FAIL"
        # Don't clean up - we need the evidence.
        find . -name core -exec chmod 0644 '{}' \;
 else

diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh
index c6eeb05be3ab6e367dcb5248993e3b9a45cbd09a..5668493b70bcb36bc2a845a0fac79231ee669487 100644 (file)
--- a/bin/tests/system/runall.sh
+++ b/bin/tests/system/runall.sh
@@ -15,38 +15,29 @@
 # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 # PERFORMANCE OF THIS SOFTWARE.
 
-#
 # Run all the system tests.
 #
-# Note: Use "make check" to run all the system tests.  This script will just
-# run those tests that require that each of their nameservers is the only one
-# running on an IP address.
+# Usage:
+#    runall.sh [numprocesses]
 #
+# ...where numprocess is the number of processes to use. The default is 1,
+# which runs the tests sequentially.
 
 SYSTEMTESTTOP=.
 . $SYSTEMTESTTOP/conf.sh
 
-status=0
-
-{
-    for d in $SEQUENTIALDIRS
-    do
-            $SHELL run.sh "${@}" $d || status=1
-    done
-} 2>&1 | tee "systests.output"
-
-$PERL testsock.pl || {
-    cat <<EOF >&2
-I:
-I:NOTE: System tests were skipped because they require that the
-I:      IP addresses 10.53.0.1 through 10.53.0.8 be configured
-I:      as alias addresses on the loopback interface.  Please run
-I:      "bin/tests/system/ifconfig.sh up" as root to configure them.
-EOF
-}
-
-echo "I:System test result summary:"
-grep '^R:' systests.output | sed -e 's/^/I: /' -e 's/R:[^:]*//' | sort | uniq -c
-grep '^R:[^:]*:FAIL' systests.output > /dev/null && status=1
-
-exit $status
+numproc=
+if [ $# -eq 0 ]; then
+    numproc=1
+
+elif [ $# -gt 1 ] ||  "$(($1 + 0))" -ne "$1" ]; then
+    echo "Usage: ./runall.sh [numprocesses]"
+
+else
+    numproc=$1
+
+fi
+
+make -j $numproc check
+
+exit $?

diff --git a/bin/tests/system/runsequential.sh b/bin/tests/system/runsequential.sh
new file mode 100644 (file)
index 0000000..caa4ed7
--- /dev/null
+++ b/bin/tests/system/runsequential.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# Copyright (C) 2000, 2001, 2004, 2007, 2010-2012, 2014-2016  Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Run system tests that must be run sequentially
+#
+# Note: Use "make check" (or runall.sh) to run all the system tests.  This
+# script will just run those tests that require that each of their nameservers
+# is the only one running on an IP address.
+#
+
+SYSTEMTESTTOP=.
+. $SYSTEMTESTTOP/conf.sh
+
+    for d in $SEQUENTIALDIRS
+    do
+        $SHELL run.sh "${@}" $d
+    done

diff --git a/bin/tests/system/testsummary.sh b/bin/tests/system/testsummary.sh
new file mode 100644 (file)
index 0000000..11cc79f
--- /dev/null
+++ b/bin/tests/system/testsummary.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+#
+# Copyright (C) 2000, 2001, 2004, 2007, 2010-2012, 2014-2016  Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Searches the system test output file (systests.output) and prints a summary
+# of tests passed, failed, not run.  It also checks whether the IP addresses
+# 10.53.0.[1-8] were set up and, if not, prints a warning.
+#
+# Usage:
+#    testsummary.sh
+#
+# Status return:
+# 0 - no tests failed
+# 1 - one or more tests failed
+
+SYSTEMTESTTOP=.
+. $SYSTEMTESTTOP/conf.sh
+
+$PERL testsock.pl || {
+    cat <<EOF >&2
+I:
+I:NOTE: System tests were skipped because they require that the
+I:      IP addresses 10.53.0.1 through 10.53.0.8 be configured
+I:      as alias addresses on the loopback interface.  Please run
+I:      "bin/tests/system/ifconfig.sh up" as root to configure them.
+EOF
+}
+
+status=0
+echo "I:System test result summary:"
+grep '^R:' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/'
+grep '^R:[^:]*:FAIL' systests.output > /dev/null && status=1
+
+exit $status