[rt46602] Tidy up run management

Miscellaneous tidying up of run management.  The most significant
change is that "runall.sh" now runs _all_ the tests, even the
ones that can run in parallel.  runsequential.sh is the script
to run tests that have not been converted to parallel running.

(cherry picked from commit 32fe6f7682361225415cc6714a1e369dca94b636)
(cherry picked from commit a49afdaeb818fb81ead578ff21a8dcd2b89d00aa)

diff --git a/bin/tests/system/Makefile.in b/bin/tests/system/Makefile.in
index 4740d38e91fd23e496363e6e3cfa912930873062..6e79ba1d133bd9c2621e6926e81c8e569940c675 100644 (file)
--- a/bin/tests/system/Makefile.in
+++ b/bin/tests/system/Makefile.in
@@ -74,24 +74,19 @@ parallel.mk:
                echo "  @$(SHELL) ./run.sh -p $$port $$directory" >> $@ ; \
        done
 
-# Targets to run the tests that can be done in parallel (which have unique
-# ports assigned to them) and the tests which must be run sequentially (because
-# they all use query port 5300 and control port 9953).
+# Targets to run the tests.
 
-parallel: parallel.mk
-       @$(MAKE) -f parallel.mk
-
-sequential:
-       @if test -f ./runall.sh; then $(SHELL) ./runall.sh; fi
-
-# Standard targets.
+test: parallel.mk
+       @$(MAKE) -f parallel.mk check 2>&1 | tee systests.output
+       @$(SHELL) ./runsequential.sh 2>&1 | tee -a systests.output
+       @$(SHELL) ./testsummary.sh
 
 check: test
 
-test: parallel sequential
+# Other targets.
 
 testclean clean distclean::
-       if test -f ./cleanall.sh; then sh ./cleanall.sh; fi
+       if test -f ./cleanall.sh; then $(SHELL) ./cleanall.sh; fi
        rm -f systests.output
        rm -f random.data
        rm -f parallel.mk

diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index 0c9af9128f654d4df19cdd7d8819ece966fb5dcd..01d55c716bfc3ec701046a3a274c85ed4a611cda 100644 (file)
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -159,7 +159,7 @@ if test -t 1 && type tput > /dev/null 2>&1 ; then
     COLOR_FAIL=`tput setaf 1`   # red
     COLOR_WARN=`tput setaf 3`   # yellow
     COLOR_PASS=`tput setaf 2`   # green
-    COLOR_STEN=`tput setaf 4`   # blue
+    COLOR_STEN=`tput setaf 4`   # blue - for start/end messages
     COLOR_INFO=`tput bold`      # bold
     COLOR_NONE=`tput sgr0`
 else
@@ -208,10 +208,6 @@ fi
 TESTNAME="`basename $PWD`"
 
 echo_i() {
-    # Without formatting
-    echo "I:$TESTNAME:$@" >> test.output
-
-    # With formatting
     echoinfo "I:$TESTNAME:$@"
 }
 

diff --git a/bin/tests/system/getopts.sh b/bin/tests/system/getopts.sh
index b3ac264b63c12c4596a2841d6deb6acf49cdf594..001cad58d69aaf53bfde31aa91b25ba71188683b 100644 (file)
--- a/bin/tests/system/getopts.sh
+++ b/bin/tests/system/getopts.sh
@@ -42,7 +42,7 @@ OPTIND=1
 # the upper of the 10 ports notionally assigned does not exceed 65535.
 
 if [ "$((${port}+0))" != "${port}" ] || [ "${port}" -le 1024 ] || [ "${port}" -gt 65520 ]; then
-    echo "Specified port '$port' must be numeric and in the range 1025 to 65520" >&2
+    echo "Base of port range ($port) must be numeric and in the range 1025 to 65520" >&2
     exit 1
 fi
 
@@ -56,6 +56,11 @@ aport7=$(($port + 7))
 aport8=$(($port + 8))
 controlport=$(($port + 9))
 
+# Two more symbols that denote the limits of the range.
+
+portlow=$port
+porthigh=$controlport
+
 
 # copy_setports - Copy Configuration File and Replace Ports
 #

diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index 5043126911f17b72a4ca2fd8569edcda13173583..f3d62b9f424773adf69342d4cb4de86c57ee85cc 100644 (file)
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -52,8 +52,7 @@ test -d $test || { echofail "$0: $test: no such test" >&2; exit 1; }
 echoinfo "S:$test:`date $dateargs`" >&2
 echoinfo "T:$test:1:A" >&2
 echoinfo "A:$test:System test $test" >&2
-echoinfo "I:$test:PORT:${port}" >&2
-echoinfo "I:$test:CONTROLPORT:${controlport}" >&2
+echoinfo "I:$test:PORTRANGE:${portlow} - ${porthigh}"
 
 if [ x${PERL:+set} = x ]
 then
@@ -122,7 +121,7 @@ $PERL stop.pl $test
 status=`expr $status + $?`
 
 if [ $status != 0 ]; then
-       echofail "R:$test:$FAIL"
+       echofail "R:$test:FAIL"
        # Don't clean up - we need the evidence.
        find . -name core -exec chmod 0644 '{}' \;
 else

diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh
index c6eeb05be3ab6e367dcb5248993e3b9a45cbd09a..5668493b70bcb36bc2a845a0fac79231ee669487 100644 (file)
--- a/bin/tests/system/runall.sh
+++ b/bin/tests/system/runall.sh
@@ -15,38 +15,29 @@
 # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 # PERFORMANCE OF THIS SOFTWARE.
 
-#
 # Run all the system tests.
 #
-# Note: Use "make check" to run all the system tests.  This script will just
-# run those tests that require that each of their nameservers is the only one
-# running on an IP address.
+# Usage:
+#    runall.sh [numprocesses]
 #
+# ...where numprocess is the number of processes to use. The default is 1,
+# which runs the tests sequentially.
 
 SYSTEMTESTTOP=.
 . $SYSTEMTESTTOP/conf.sh
 
-status=0
-
-{
-    for d in $SEQUENTIALDIRS
-    do
-            $SHELL run.sh "${@}" $d || status=1
-    done
-} 2>&1 | tee "systests.output"
-
-$PERL testsock.pl || {
-    cat <<EOF >&2
-I:
-I:NOTE: System tests were skipped because they require that the
-I:      IP addresses 10.53.0.1 through 10.53.0.8 be configured
-I:      as alias addresses on the loopback interface.  Please run
-I:      "bin/tests/system/ifconfig.sh up" as root to configure them.
-EOF
-}
-
-echo "I:System test result summary:"
-grep '^R:' systests.output | sed -e 's/^/I: /' -e 's/R:[^:]*//' | sort | uniq -c
-grep '^R:[^:]*:FAIL' systests.output > /dev/null && status=1
-
-exit $status
+numproc=
+if [ $# -eq 0 ]; then
+    numproc=1
+
+elif [ $# -gt 1 ] ||  "$(($1 + 0))" -ne "$1" ]; then
+    echo "Usage: ./runall.sh [numprocesses]"
+
+else
+    numproc=$1
+
+fi
+
+make -j $numproc check
+
+exit $?

diff --git a/bin/tests/system/runsequential.sh b/bin/tests/system/runsequential.sh
new file mode 100644 (file)
index 0000000..caa4ed7
--- /dev/null
+++ b/bin/tests/system/runsequential.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# Copyright (C) 2000, 2001, 2004, 2007, 2010-2012, 2014-2016  Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Run system tests that must be run sequentially
+#
+# Note: Use "make check" (or runall.sh) to run all the system tests.  This
+# script will just run those tests that require that each of their nameservers
+# is the only one running on an IP address.
+#
+
+SYSTEMTESTTOP=.
+. $SYSTEMTESTTOP/conf.sh
+
+    for d in $SEQUENTIALDIRS
+    do
+        $SHELL run.sh "${@}" $d
+    done

diff --git a/bin/tests/system/testsummary.sh b/bin/tests/system/testsummary.sh
new file mode 100644 (file)
index 0000000..11cc79f
--- /dev/null
+++ b/bin/tests/system/testsummary.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+#
+# Copyright (C) 2000, 2001, 2004, 2007, 2010-2012, 2014-2016  Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Searches the system test output file (systests.output) and prints a summary
+# of tests passed, failed, not run.  It also checks whether the IP addresses
+# 10.53.0.[1-8] were set up and, if not, prints a warning.
+#
+# Usage:
+#    testsummary.sh
+#
+# Status return:
+# 0 - no tests failed
+# 1 - one or more tests failed
+
+SYSTEMTESTTOP=.
+. $SYSTEMTESTTOP/conf.sh
+
+$PERL testsock.pl || {
+    cat <<EOF >&2
+I:
+I:NOTE: System tests were skipped because they require that the
+I:      IP addresses 10.53.0.1 through 10.53.0.8 be configured
+I:      as alias addresses on the loopback interface.  Please run
+I:      "bin/tests/system/ifconfig.sh up" as root to configure them.
+EOF
+}
+
+status=0
+echo "I:System test result summary:"
+grep '^R:' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/'
+grep '^R:[^:]*:FAIL' systests.output > /dev/null && status=1
+
+exit $status