From: Stephen Morris <stephen@isc.org>
Date: Fri, 17 Nov 2017 13:21:05 +0000 (+0000)
Subject: [rt46602] Ensure that tests running in parallel use unique ports
X-Git-Tag: v9.10.8rc1~84^2~43
X-Git-Url: http://git.ipfire.org/gitweb/?a=commitdiff_plain;h=704e2324e3ade6b5a19c6391a5230924715b1544;p=thirdparty%2Fbind9.git

[rt46602] Ensure that tests running in parallel use unique ports

Via an intermediate make file, tests that have been modified to be able
to run in parallel are assigned unique query and control port numbers
(other than 5300 and 9953 respectively).  Tests that have not yet been
modified all use ports 5300 and 9953, so must be run sequentially.

(cherry picked from commit e7429b124bd3d0fcce107eaf790ece4ba332eccd)
(cherry picked from commit 4f68e67e5cb5799cf26657f3e00ae3fca636a96f)
---

diff --git a/bin/tests/system/Makefile.in b/bin/tests/system/Makefile.in
index 9dbd6a0a4d9..f52d2effe49 100644
--- a/bin/tests/system/Makefile.in
+++ b/bin/tests/system/Makefile.in
@@ -47,26 +47,53 @@ TARGETS =	feature-test@EXEEXT@
 feature-test@EXEEXT@: feature-test.@O@
 	${LIBTOOL_MODE_LINK} ${PURIFY} ${CC} ${CFLAGS} ${LDFLAGS} -o $@ feature-test.@O@ ${ISCLIBS} ${LIBS}
 
-# Running the scripts below is bypassed when a separate
-# build directory is used.
 
-check: test
+# Running the scripts below is bypassed when a separate # build directory is
+# used.
+
+# Define the tests that can be run in parallel.  This should be identical to
+# the definition of PARALLELDIRS in conf.sh.  These tests do not use query
+# port 5300 or control port 9953.
+
+PARALLEL = allow_query serve-stale rpzrecurse
+
+# Produce intermediate makefile that assigns unique port numbers to each
+# parallel test.
+
+parallel.mk:
+	@echo ".PHONY: $(PARALLEL)" > $@ ; \
+	echo "" >> $@ ; \
+	echo "check: $(PARALLEL)" >> $@ ; \
+	port=5299 ; \
+	for directory in $(PARALLEL) ; do \
+		port=$$(($$port + 2)) ; \
+		controlport=$$(($$port + 1)) ; \
+		echo "" >> $@ ; \
+		echo "$$directory:" >> $@ ; \
+		echo "	@$(SHELL) ./run.sh -p $$port -c $$controlport $$directory" >> $@ ; \
+	done
 
-TESTS = allow_query serve-stale rpzrecurse alltests
+# Targets to run the tests that can be done in parallel (which have unique
+# ports assigned to them) and the tests which must be run sequentially (because
+# they all use query port 5300 and control port 9953).
 
-test: subdirs $(TESTS)
+parallel: parallel.mk
+	$(MAKE) -f parallel.mk
 
-alltests:
+sequential:
 	if test -f ./runall.sh; then $(SHELL) ./runall.sh; fi
 
-$(TESTS):
-	PORT=$$((32767+$$RANDOM)); \
-	$(SHELL) ./run.sh -p $$(($$PORT)) -c $$(($$PORT+1)) $@
+# Standard targets.
+
+check: test
+
+test: parallel sequential
 
 testclean clean distclean::
 	if test -f ./cleanall.sh; then sh ./cleanall.sh; fi
 	rm -f systests.output
 	rm -f random.data
+	rm -f parallel.mk
 	rm -f ${TARGETS}
 	rm -f ${OBJS}
 
@@ -78,5 +105,3 @@ installdirs:
 install::
 
 uninstall::
-
-.PHONY: $(TESTS)
diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index 437d482ee49..e011999133f 100644
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -76,12 +76,16 @@ KRB5_CONFIG=/dev/null
 # The "stress" test is not run by default since it creates enough
 # load on the machine to make it unusable to other users.
 # v6synth
-SUBDIRS="acl additional allow_query addzone autosign builtin
-	 cacheclean case chain checkconf @CHECKDS@ checknames
-	 checkzone cookie @COVERAGE@ database delv digdelv dlv
-	 dlz dlzexternal dlzredir dns64 dnssec dsdigest
-	 dscp ecdsa eddsa ednscompliance emptyzones fetchlimit filter-aaaa
-	 formerr forward geoip glue gost ixfr inline integrity
+#
+# List of tests that use ports 5300 and 9953.  For this reason, these must
+# be run sequentially.
+SEQUENTIALDIRS="acl additional addzone auth autosign builtin
+	 cacheclean case cds chain
+         checkconf @CHECKDS@ checknames checkzone cookie @COVERAGE@
+         database digdelv dlv dlz dlzexternal
+	 dns64 dnssec @DNSTAP@ dscp dsdigest dyndb ecdsa eddsa
+	 ednscompliance emptyzones fetchlimit filter-aaaa formerr
+	 forward geoip glue gost inline integrity ixfr
 	 legacy limits logfileconfig lwresd masterfile masterformat
 	 metadata notify nslookup nsupdate pending @PKCS11_TEST@
 	 reclimit redirect resolver rndc rpz rpzrecurse rrl rrchecker
@@ -89,6 +93,15 @@ SUBDIRS="acl additional allow_query addzone autosign builtin
 	 statistics statschannel stub tcp tkey tsig tsiggss unknown
 	 upforwd verify views wildcard xfer xferquota zero zonechecks"
 
+# List of tests that use unique ports (other than 5300 and 9953). These
+# tests can be run in parallel.
+#
+# This symbol must be kept in step with the PARALLEL macro in Makefile.in
+PARALLELDIRS="allow_query rps-recurse serve-stale"
+
+SUBDIRS="$SEQUENTIALDIRS $PARALLELDIRS"
+
+
 # Things that are different on Windows
 KILL=kill
 DIFF=diff
@@ -189,6 +202,10 @@ fi
 TESTNAME="`basename $PWD`"
 
 echo_i() {
+    # Without formatting
+    echo "I:$TESTNAME:$@" >> test.output
+
+    # With formatting
     echoinfo "I:$TESTNAME:$@"
 }
 
diff --git a/bin/tests/system/conf.sh.win32 b/bin/tests/system/conf.sh.win32
index ff8bcf3384b..581034bd0bf 100644
--- a/bin/tests/system/conf.sh.win32
+++ b/bin/tests/system/conf.sh.win32
@@ -88,7 +88,10 @@ KRB5_CONFIG=NUL
 # The "stress" test is not run by default since it creates enough
 # load on the machine to make it unusable to other users.
 # v6synth
-SUBDIRS="acl additional addzone autosign builtin
+#
+# List of tests that use ports 5300 and 9953.  For this reason, these must
+# be run sequentially.
+SEQUENTIALDIRS="acl additional addzone autosign builtin
 	 cacheclean case
 	 checkconf @CHECKDS@ checknames checkzone cookie @COVERAGE@
 	 database digdelv dlv dlvauto dlz dlzexternal
@@ -103,6 +106,17 @@ SUBDIRS="acl additional addzone autosign builtin
 	 tkey tsig tsiggss unknown upforwd verify views wildcard xfer
 	 xferquota zero zonechecks"
 
+# List of tests that use unique ports (other than 5300 and 9953). These
+# tests can be run in parallel.
+#
+# This symbol must be kept in step with the PARALLEL macro in Makefile.in
+PARALLELDIRS="allow_query rpzrecurse serve-stale"
+
+SUBDIRS="$SEQUENTIALDIRS $PARALLELDIRS"
+
+# missing: chain integrity
+# extra: dname ednscompliance forward
+
 #Things that are different on Windows
 KILL="/bin/kill -f"
 DIFF="diff --strip-trailing-cr"
diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh
index c746f0ce470..c6eeb05be3a 100644
--- a/bin/tests/system/runall.sh
+++ b/bin/tests/system/runall.sh
@@ -18,6 +18,10 @@
 #
 # Run all the system tests.
 #
+# Note: Use "make check" to run all the system tests.  This script will just
+# run those tests that require that each of their nameservers is the only one
+# running on an IP address.
+#
 
 SYSTEMTESTTOP=.
 . $SYSTEMTESTTOP/conf.sh
@@ -25,7 +29,7 @@ SYSTEMTESTTOP=.
 status=0
 
 {
-    for d in $SUBDIRS
+    for d in $SEQUENTIALDIRS
     do
             $SHELL run.sh "${@}" $d || status=1
     done