From: Stephen Morris <stephen@isc.org>
Date: Thu, 23 Nov 2017 10:02:15 +0000 (+0000)
Subject: [rt46602] Tidy up run management
X-Git-Tag: v9.13.0~158^2~42
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=32fe6f7682361225415cc6714a1e369dca94b636;p=thirdparty%2Fbind9.git

[rt46602] Tidy up run management

Miscellaneous tidying up of run management.  The most significant
change is that "runall.sh" now runs _all_ the tests, even the
ones that can run in parallel.  runsequential.sh is the script
to run tests that have not been converted to parallel running.
---

diff --git a/bin/tests/system/Makefile.in b/bin/tests/system/Makefile.in
index e27a2a04f04..f14118acc4e 100644
--- a/bin/tests/system/Makefile.in
+++ b/bin/tests/system/Makefile.in
@@ -66,24 +66,19 @@ parallel.mk:
 		echo "	@$(SHELL) ./run.sh -p $$port $$directory" >> $@ ; \
 	done
 
-# Targets to run the tests that can be done in parallel (which have unique
-# ports assigned to them) and the tests which must be run sequentially (because
-# they all use query port 5300 and control port 9953).
+# Targets to run the tests.
 
-parallel: parallel.mk
-	@$(MAKE) -f parallel.mk
-
-sequential:
-	@if test -f ./runall.sh; then $(SHELL) ./runall.sh; fi
-
-# Standard targets.
+test: parallel.mk
+	@$(MAKE) -f parallel.mk check 2>&1 | tee systests.output
+	@$(SHELL) ./runsequential.sh 2>&1 | tee -a systests.output
+	@$(SHELL) ./testsummary.sh
 
 check: test
 
-test: parallel sequential
+# Other targets.
 
 testclean clean distclean::
-	if test -f ./cleanall.sh; then sh ./cleanall.sh; fi
+	if test -f ./cleanall.sh; then $(SHELL) ./cleanall.sh; fi
 	rm -f systests.output
 	rm -f random.data
 	rm -f parallel.mk
diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index 924ba4a8e31..8646d6cff02 100644
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -160,7 +160,7 @@ if test -t 1 && type tput > /dev/null 2>&1 ; then
     COLOR_FAIL=`tput setaf 1`   # red
     COLOR_WARN=`tput setaf 3`   # yellow
     COLOR_PASS=`tput setaf 2`   # green
-    COLOR_STEN=`tput setaf 4`   # blue
+    COLOR_STEN=`tput setaf 4`   # blue - for start/end messages
     COLOR_INFO=`tput bold`      # bold
     COLOR_NONE=`tput sgr0`
 else
@@ -209,10 +209,6 @@ fi
 TESTNAME="`basename $PWD`"
 
 echo_i() {
-    # Without formatting
-    echo "I:$TESTNAME:$@" >> test.output
-
-    # With formatting
     echoinfo "I:$TESTNAME:$@"
 }
 
diff --git a/bin/tests/system/getopts.sh b/bin/tests/system/getopts.sh
index 24d1b367319..e2e8455f09c 100644
--- a/bin/tests/system/getopts.sh
+++ b/bin/tests/system/getopts.sh
@@ -42,7 +42,7 @@ OPTIND=1
 # the upper of the 10 ports notionally assigned does not exceed 65535.
 
 if [ "$((${port}+0))" != "${port}" ] || [ "${port}" -le 1024 ] || [ "${port}" -gt 65520 ]; then
-    echo "Specified port '$port' must be numeric and in the range 1025 to 65520" >&2
+    echo "Base of port range ($port) must be numeric and in the range 1025 to 65520" >&2
     exit 1
 fi
 
@@ -56,6 +56,11 @@ aport7=$(($port + 7))
 aport8=$(($port + 8))
 controlport=$(($port + 9))
 
+# Two more symbols that denote the limits of the range.
+
+portlow=$port
+porthigh=$controlport
+
 
 # copy_setports - Copy Configuration File and Replace Ports
 #
diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index 99dcdbccb0f..68bc86c1877 100644
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -43,8 +43,7 @@ test -d $test || { echofail "$0: $test: no such test" >&2; exit 1; }
 echoinfo "S:$test:`date $dateargs`" >&2
 echoinfo "T:$test:1:A" >&2
 echoinfo "A:$test:System test $test" >&2
-echoinfo "I:$test:PORT:${port}" >&2
-echoinfo "I:$test:CONTROLPORT:${controlport}" >&2
+echoinfo "I:$test:PORTRANGE:${portlow} - ${porthigh}"
 
 if [ x${PERL:+set} = x ]
 then
@@ -113,7 +112,7 @@ $PERL stop.pl $test
 status=`expr $status + $?`
 
 if [ $status != 0 ]; then
-	echofail "R:$test:$FAIL"
+	echofail "R:$test:FAIL"
 	# Don't clean up - we need the evidence.
 	find . -name core -exec chmod 0644 '{}' \;
 else
diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh
index 315caf67c0f..f9e1e47a3cb 100644
--- a/bin/tests/system/runall.sh
+++ b/bin/tests/system/runall.sh
@@ -6,38 +6,29 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-#
 # Run all the system tests.
 #
-# Note: Use "make check" to run all the system tests.  This script will just
-# run those tests that require that each of their nameservers is the only one
-# running on an IP address.
+# Usage:
+#    runall.sh [numprocesses]
 #
+# ...where numprocess is the number of processes to use. The default is 1,
+# which runs the tests sequentially.
 
 SYSTEMTESTTOP=.
 . $SYSTEMTESTTOP/conf.sh
 
-status=0
-
-{
-    for d in $SEQUENTIALDIRS
-    do
-            $SHELL run.sh "${@}" $d || status=1
-    done
-} 2>&1 | tee "systests.output"
-
-$PERL testsock.pl || {
-    cat <<EOF >&2
-I:
-I:NOTE: System tests were skipped because they require that the
-I:      IP addresses 10.53.0.1 through 10.53.0.8 be configured
-I:      as alias addresses on the loopback interface.  Please run
-I:      "bin/tests/system/ifconfig.sh up" as root to configure them.
-EOF
-}
-
-echo "I:System test result summary:"
-grep '^R:' systests.output | sed -e 's/^/I: /' -e 's/R:[^:]*//' | sort | uniq -c
-grep '^R:[^:]*:FAIL' systests.output > /dev/null && status=1
-
-exit $status
+numproc=
+if [ $# -eq 0 ]; then
+    numproc=1
+
+elif [ $# -gt 1 ] ||  "$(($1 + 0))" -ne "$1" ]; then
+    echo "Usage: ./runall.sh [numprocesses]"
+
+else
+    numproc=$1
+
+fi
+
+make -j $numproc check
+
+exit $?
diff --git a/bin/tests/system/runsequential.sh b/bin/tests/system/runsequential.sh
new file mode 100644
index 00000000000..caa4ed7b93e
--- /dev/null
+++ b/bin/tests/system/runsequential.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# Copyright (C) 2000, 2001, 2004, 2007, 2010-2012, 2014-2016  Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Run system tests that must be run sequentially
+#
+# Note: Use "make check" (or runall.sh) to run all the system tests.  This
+# script will just run those tests that require that each of their nameservers
+# is the only one running on an IP address.
+#
+
+SYSTEMTESTTOP=.
+. $SYSTEMTESTTOP/conf.sh
+
+    for d in $SEQUENTIALDIRS
+    do
+        $SHELL run.sh "${@}" $d
+    done
diff --git a/bin/tests/system/testsummary.sh b/bin/tests/system/testsummary.sh
new file mode 100644
index 00000000000..11cc79f12c8
--- /dev/null
+++ b/bin/tests/system/testsummary.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+#
+# Copyright (C) 2000, 2001, 2004, 2007, 2010-2012, 2014-2016  Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Searches the system test output file (systests.output) and prints a summary
+# of tests passed, failed, not run.  It also checks whether the IP addresses
+# 10.53.0.[1-8] were set up and, if not, prints a warning.
+#
+# Usage:
+#    testsummary.sh
+#
+# Status return:
+# 0 - no tests failed
+# 1 - one or more tests failed
+
+SYSTEMTESTTOP=.
+. $SYSTEMTESTTOP/conf.sh
+
+$PERL testsock.pl || {
+    cat <<EOF >&2
+I:
+I:NOTE: System tests were skipped because they require that the
+I:      IP addresses 10.53.0.1 through 10.53.0.8 be configured
+I:      as alias addresses on the loopback interface.  Please run
+I:      "bin/tests/system/ifconfig.sh up" as root to configure them.
+EOF
+}
+
+status=0
+echo "I:System test result summary:"
+grep '^R:' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/'
+grep '^R:[^:]*:FAIL' systests.output > /dev/null && status=1
+
+exit $status