Bail-out early if dig fails to finish successfully or takes too long

Before, the zero system test could get stuck almost infinitely, because
the first test sends > 300 queries with 5 seconds timeout on each in
each pass.  If named crashed early, it would took the test more than 4
hours to properly timeout.

This commit introduces a "watchdog" on the dig commands running in the
background and failing the test on timeout, failing any test if any dig
command fails to return successfully, and making the tests.sh script
shellcheck clean.

diff --git a/bin/tests/system/zero/tests.sh b/bin/tests/system/zero/tests.sh
index b89aee223d65124627b536cecfa7e1dbac378404..681ba118db3a9a7becb7eba9f7dc9427de529a4f 100644 (file)
--- a/bin/tests/system/zero/tests.sh
+++ b/bin/tests/system/zero/tests.sh
@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 # Copyright (C) Internet Systems Consortium, Inc. ("ISC")
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
@@ -7,84 +9,112 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+#shellcheck source=conf.sh
 SYSTEMTESTTOP=..
-. $SYSTEMTESTTOP/conf.sh
+. "$SYSTEMTESTTOP/conf.sh"
+
+dig_with_opts() {
+       "$DIG" -p "${PORT}" "$@"
+}
 
-DIGOPTS="-p ${PORT}"
+wait_for_pid() (
+       for pid in "$@"; do
+               kill -0 "$pid" 2>/dev/null && return 1
+       done
+       return 0
+)
 
 status=0
 n=0
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check lookups against TTL=0 records ($n)"
 i=0
+ret=0
 passes=10
-$DIG $DIGOPTS @10.53.0.2 axfr example | grep -v "^ds0" |
+dig_with_opts @10.53.0.2 axfr example | grep -v "^ds0" | \
 awk '$2 == "0" { print "-q", $1, $4; print "-q", "zzz"$1, $4;}' > query.list
+
+# add 1/5 second per query
+timeout=$(($(wc -l < query.list) / 5))
 while [ $i -lt $passes ]
 do
-       ret=0
-       $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.1.test$n &
-       $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.2.test$n &
-       $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.3.test$n &
-       $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.4.test$n &
-       $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.5.test$n &
-       $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.6.test$n &
-       wait
-       grep "status: SERVFAIL" dig.out$i.1.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.2.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.3.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.4.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.5.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.6.test$n > /dev/null && ret=1
+       (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.1.test$n") & pid1="$!"
+       (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.2.test$n") & pid2="$!"
+       (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.3.test$n") & pid3="$!"
+       (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.4.test$n") & pid4="$!"
+       (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.5.test$n") & pid5="$!"
+       (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.6.test$n") & pid6="$!"
+
+       retry_quiet "$timeout" wait_for_pid "$pid1" "$pid2" "$pid3" "$pid4" "$pid5" "$pid6" || ret=1
+       kill -TERM "$pid1" "$pid2" "$pid3" "$pid4" "$pid5" "$pid6" 2>/dev/null
+
+       wait "$pid1" || ret=1
+       wait "$pid2" || ret=1
+       wait "$pid3" || ret=1
+       wait "$pid4" || ret=1
+       wait "$pid5" || ret=1
+       wait "$pid6" || ret=1
+
+       grep "status: SERVFAIL" "dig.out$i.1.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.2.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.3.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.4.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.5.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.6.test$n" > /dev/null && ret=1
        [ $ret = 1 ] && break
-       i=`expr $i + 1`
+       i=$((i+1))
        echo_i "successfully completed pass $i of $passes"
 done
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
+
+repeat_query() (
+       i=0
+       while [ "$i" -lt "$1" ]; do
+               dig_with_opts +short "@$2" "$3" | tee "dig.out$i.test$n" || return 1
+               i=$((i+1))
+       done
+)
 
-n=`expr $n + 1`
+count_unique() (
+       repeat_query "$@" | sort -u | wc -l
+)
+
+n=$((n+1))
 echo_i "check repeated recursive lookups of non recurring TTL=0 responses get new values ($n)"
 ret=0
-count=`(
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-) | sort -u | wc -l `
-if [ $count -ne 7 ] ; then echo_i "failed (count=$count)"; ret=1; fi
-status=`expr $status + $ret`
+repeats=9
+count=$(count_unique "$repeats" 10.53.0.3 foo.increment)
+if [ "$count" -ne "$repeats" ] ; then echo_i "failed (count=$count, repeats=$repeats)"; ret=1; fi
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check lookups against TTL=1 records ($n)"
 i=0
 passes=10
+ret=0
 while [ $i -lt $passes ]
 do
-       ret=0
-       $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.1.test$n
-       $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.2.test$n
-       $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.3.test$n
-       $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.4.test$n
-       $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.5.test$n
-       $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.6.test$n
-       grep "status: SERVFAIL" dig.out$i.1.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.2.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.3.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.4.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.5.test$n > /dev/null && ret=1
-       grep "status: SERVFAIL" dig.out$i.6.test$n > /dev/null && ret=1
+       dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.1.test$n" || ret=1
+       dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.2.test$n" || ret=1
+       dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.3.test$n" || ret=1
+       dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.4.test$n" || ret=1
+       dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.5.test$n" || ret=1
+       dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.6.test$n" || ret=1
+       grep "status: SERVFAIL" "dig.out$i.1.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.2.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.3.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.4.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.5.test$n" > /dev/null && ret=1
+       grep "status: SERVFAIL" "dig.out$i.6.test$n" > /dev/null && ret=1
        [ $ret = 1 ] && break
-       i=`expr $i + 1`
+       i=$((i+1))
        echo_i "successfully completed pass $i of $passes"
-       $PERL -e 'select(undef, undef, undef, 0.3);'
+       sleep 1
 done
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
 echo_i "exit status: $status"
-[ $status -eq 0 ] || exit 1
+[ "$status" -eq 0 ] || exit 1