tests: addzone: retry when checking for things, to allow for timing problems

diff --git a/bin/tests/system/addzone/clean.sh b/bin/tests/system/addzone/clean.sh
index 30c349c830107fc3fc5c7c20d160bb3e9a256bb5..de727c308e7d61b00d40e28612934d5bb2f546a0 100644 (file)
--- a/bin/tests/system/addzone/clean.sh
+++ b/bin/tests/system/addzone/clean.sh
@@ -9,32 +9,34 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
-rm -f dig.out.*
-rm -f rndc.out*
-rm -f showzone.out*
-rm -f zonestatus.out*
-rm -f */named.conf
-rm -f */named.memstats
-rm -f ns1/*.nzf ns1/*.nzf~
-rm -f ns1/*.nzd ns1/*.nzd-lock
-rm -f ns2/*.nzf ns2/*.nzf~
-rm -f ns2/*.nzd ns2/*.nzd-lock
-rm -f ns3/*.nzf ns3/*.nzf~
-rm -f ns3/*.nzd ns3/*.nzd-lock
-rm -f ns2/core*
-rm -f ns2/inline.db.jbk
-rm -f ns2/inline.db.signed
-rm -f ns2/inlineslave.bk*
-rm -rf ns2/new-zones
-rm -f ns*/named.lock
-rm -f ns*/named.run
-rm -f ns2/nzf-*
-rm -f ns3/named.conf
-rm -f ns3/*.nzf ns3/*.nzf~
-rm -f ns3/*.nzd ns3/*.nzd-lock
-rm -f ns3/inlineslave.db
-rm -f ns1/redirect.db
-rm -f ns2/redirect.db
-rm -f ns2/redirect.bk
-rm -f ns3/redirect.db
-rm -f ns*/managed-keys.bind* ns*/*.mkeys*
+rm -f ./dig.out.*
+rm -f ./rndc.out*
+rm -f ./showzone.out*
+rm -f ./zonestatus.out*
+rm -f ./*/named.conf
+rm -f ./*/named.memstats
+rm -f ./ns1/*.nzf ./ns1/*.nzf~
+rm -f ./ns1/*.nzd ./ns1/*.nzd-lock
+rm -f ./ns2/*.nzf ./ns2/*.nzf~
+rm -f ./ns2/*.nzd ./ns2/*.nzd-lock
+rm -f ./ns3/*.nzf ./ns3/*.nzf~
+rm -f ./ns3/*.nzd ./ns3/*.nzd-lock
+rm -f ./ns2/core*
+rm -f ./ns2/inline.db.jbk
+rm -f ./ns2/inline.db.signed
+rm -f ./ns2/inlineslave.bk*
+rm -rf ./ns2/new-zones
+rm -f ./ns*/named.lock
+rm -f ./ns*/named.run ./ns*/named.run.prev
+rm -f ./ns2/nzf-*
+rm -f ./ns3/named.conf
+rm -f ./ns3/*.nzf ./ns3/*.nzf~
+rm -f ./ns3/*.nzd ns3/*.nzd-lock
+rm -f ./ns3/inlineslave.db
+rm -f ./ns1/redirect.db
+rm -f ./ns2/redirect.db
+rm -f ./ns2/redirect.bk
+rm -f ./ns3/redirect.db
+rm -f ./ns*/managed-keys.bind* ns*/*.mkeys*
+rm -f ./nzd2nzf.out.*
+rm -f ./wait_for_message.*

diff --git a/bin/tests/system/addzone/tests.sh b/bin/tests/system/addzone/tests.sh
index 7019dab931620a308e38449ce4a18ad1c922834b..4ce96bcfdbe4a25a7462133e31867d759082aee6 100755 (executable)
--- a/bin/tests/system/addzone/tests.sh
+++ b/bin/tests/system/addzone/tests.sh
@@ -55,25 +55,30 @@ fi
 echo_i "adding new zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone 'added.example { type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.$n > /dev/null || ret=1
+_check_adding_new_zone () (
+       $DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n &&
+       grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+       grep '^a.added.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_adding_new_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
+nextpart ns2/named.run >/dev/null
 echo_i "checking addzone errors are logged correctly"
 ret=0
 $RNDCCMD 10.53.0.2 addzone bad.example '{ type mister; };' 2>&1 | grep 'unexpected token' > /dev/null 2>&1 || ret=1
-grep "addzone: 'mister' unexpected" ns2/named.run >/dev/null 2>&1 || ret=1
+wait_for_log_peek 20 "addzone: 'mister' unexpected" ns2/named.run || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
+nextpart ns2/named.run >/dev/null
 echo_i "checking modzone errors are logged correctly"
 ret=0
 $RNDCCMD 10.53.0.2 modzone added.example '{ type mister; };' 2>&1 | grep 'unexpected token' > /dev/null 2>&1 || ret=1
-grep "modzone: 'mister' unexpected" ns2/named.run >/dev/null 2>&1 || ret=1
+wait_for_log_peek 20 "modzone: 'mister' unexpected" ns2/named.run || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -81,9 +86,12 @@ status=`expr $status + $ret`
 echo_i "adding a zone that requires quotes ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"32/1.0.0.127-in-addr.added.example" { check-names ignore; type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 "a.32/1.0.0.127-in-addr.added.example" a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.32/1.0.0.127-in-addr.added.example' dig.out.ns2.$n > /dev/null || ret=1
+_check_zone_that_requires_quotes() (
+       $DIG $DIGOPTS @10.53.0.2 "a.32/1.0.0.127-in-addr.added.example" a > dig.out.ns2.$n &&
+       grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+       grep '^a.32/1.0.0.127-in-addr.added.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_zone_that_requires_quotes || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -91,9 +99,12 @@ status=`expr $status + $ret`
 echo_i "adding a zone with a quote in the name ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"foo\"bar.example" { check-names ignore; type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.foo\\"bar.example' dig.out.ns2.$n > /dev/null || ret=1
+_check_zone_with_a_quote() (
+       $DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n &&
+       grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+       grep '^a.foo\\"bar.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_zone_with_a_quote || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -141,21 +152,24 @@ fi
 echo_i "deleting previously added zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone previous.example 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 a.previous.example a > dig.out.ns2.$n
-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.previous.example' dig.out.ns2.$n > /dev/null && ret=1
+_check_deleting_previously_added_zone() (
+       $DIG $DIGOPTS @10.53.0.2 a.previous.example a > dig.out.ns2.$n &&
+       grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+       ! grep '^a.previous.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_previously_added_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
+check_nzd2nzf() (
+       $NZD2NZF ns2/_default.nzd > nzd2nzf.out.$n &&
+       ! grep previous.example nzd2nzf.out.$n > /dev/null
+)
+
 if [ -n "$NZD" ]; then
     echo_i "checking zone was deleted from NZD ($n)"
-    for i in 0 1 2 3 4 5 6 7 8 9; do
-        ret=0
-        $NZD2NZF ns2/_default.nzd | grep previous.example > /dev/null && ret=1
-        [ $ret = 0 ] && break
-        sleep 1
-    done
+    retry_quiet 10 check_nzd2nzf || ret=1
     if [ $ret != 0 ]; then echo_i "failed"; fi
     status=`expr $status + $ret`
 fi
@@ -173,9 +187,12 @@ fi
 echo_i "deleting newly added zone added.example ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone added.example 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n
-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.$n > /dev/null && ret=1
+_check_deleting_newly_added_zone() (
+       $DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n &&
+       grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+       ! grep '^a.added.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_newly_added_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -183,9 +200,12 @@ status=`expr $status + $ret`
 echo_i "deleting newly added zone with escaped quote ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone "foo\\\"bar.example" 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n
-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
-grep "^a.foo\"bar.example" dig.out.ns2.$n > /dev/null && ret=1
+_check_deleting_newly_added_zone_quote() (
+       $DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n &&
+       grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+       ! grep "^a.foo\"bar.example" dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_newly_added_zone_quote || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -231,7 +251,7 @@ ret=0
 sleep 1
 cp -f ns1/redirect.db.2 ns1/redirect.db
 $RNDCCMD 10.53.0.1 reload -redirect > rndc.out.ns1.$n
-retry 5 check_zonestatus 1 || ret=1
+retry_quiet 5 check_zonestatus 1 || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -239,27 +259,27 @@ status=`expr $status + $ret`
 echo_i "delete a normally-loaded zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone normal.example > rndc.out.ns2.$n 2>&1
-$DIG $DIGOPTS @10.53.0.2 a.normal.example a > dig.out.ns2.$n
-grep "is no longer active and will be deleted" rndc.out.ns2.$n > /dev/null || ret=1
+grep "is no longer active and will be deleted" rndc.out.ns2.$n > /dev/null || ret=11
 grep "To keep it from returning when the server is restarted" rndc.out.ns2.$n > /dev/null || ret=1
 grep "must also be removed from named.conf." rndc.out.ns2.$n > /dev/null || ret=1
+_check_delete_normally_loaded_zone() (
+       $DIG $DIGOPTS @10.53.0.2 a.normal.example a > dig.out.ns2.$n &&
+       grep 'status: REFUSED' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 5 _check_delete_normally_loaded_zone || ret=1
 
-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
 echo_i "attempting to add master zone with inline signing ($n)"
 $RNDCCMD 10.53.0.2 addzone 'inline.example { type master; file "inline.db"; inline-signing yes; };' 2>&1 | sed 's/^/I:ns2 /'
-for i in 1 2 3 4 5
-do
-ret=0
-$DIG $DIGOPTS @10.53.0.2 a.inline.example a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.inline.example' dig.out.ns2.$n > /dev/null || ret=1
-[ $ret = 0 ] && break
-sleep 1
-done
+_check_add_master_zone_with_inline() (
+       $DIG $DIGOPTS @10.53.0.2 a.inline.example a > dig.out.ns2.$n &&
+       grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+       grep '^a.inline.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 5 _check_add_master_zone_with_inline || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -274,27 +294,20 @@ status=`expr $status + $ret`
 
 echo_i "attempting to add slave zone with inline signing ($n)"
 $RNDCCMD 10.53.0.2 addzone 'inlineslave.example { type slave; masters { 10.53.0.1; }; file "inlineslave.bk"; inline-signing yes; };' 2>&1 | sed 's/^/I:ns2 /'
-for i in 1 2 3 4 5
-do
-ret=0
-$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null || ret=1
-[ $ret = 0 ] && break
-sleep 1
-done
+_check_add_slave_with_inline() (
+       $DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n &&
+       grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+       grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 5 _check_add_slave_with_inline || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
 echo_i "attempting to delete slave zone with inline signing ($n)"
 ret=0
-for i in 0 1 2 3 4 5 6 7 8 9
-do
-       test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk && break
-       sleep 1
-done
-$RNDCCMD 10.53.0.2 delzone inlineslave.example 2>&1 > rndc.out2.test$n
+retry_quiet 10 test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk || ret=1
+$RNDCCMD 10.53.0.2 delzone inlineslave.example > rndc.out2.test$n 2>&1 || ret=1
 test -f inlineslave.bk ||
 grep '^inlineslave.bk$' rndc.out2.test$n > /dev/null || {
        echo_i "failed to report inlineslave.bk"; ret=1;
@@ -308,43 +321,30 @@ status=`expr $status + $ret`
 
 echo_i "restoring slave zone with inline signing ($n)"
 $RNDCCMD 10.53.0.2 addzone 'inlineslave.example { type slave; masters { 10.53.0.1; }; file "inlineslave.bk"; inline-signing yes; };' 2>&1 | sed 's/^/I:ns2 /'
-for i in 1 2 3 4 5
-do
-ret=0
-$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null || ret=1
-[ $ret = 0 ] && break
-sleep 1
-done
+_check_restoring_slave_with_inline() (
+       $DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n &&
+       grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+       grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 5 _check_restoring_slave_with_inline || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
 echo_i "deleting slave zone with automatic zone file removal ($n)"
 ret=0
-for i in 0 1 2 3 4 5 6 7 8 9
-do
-       test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk && break
-       sleep 1
-done
-$RNDCCMD 10.53.0.2 delzone -clean inlineslave.example 2>&1 > /dev/null
-for i in 0 1 2 3 4 5 6 7 8 9
-do
-        ret=0
-       test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk && ret=1
-        [ $ret = 0 ] && break
-       sleep 1
-done
+retry_quiet 10 test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk || ret=1
+$RNDCCMD 10.53.0.2 delzone -clean inlineslave.example > /dev/null 2>&1
+retry_quiet 10 test ! -f ns2/inlineslave.bk.signed -a ! -f ns2/inlineslave.bk
 n=`expr $n + 1`
 status=`expr $status + $ret`
 
 echo_i "modifying zone configuration ($n)"
 ret=0
-$RNDCCMD 10.53.0.2 addzone 'mod.example { type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
+$RNDCCMD 10.53.0.2 addzone 'mod.example { type master; file "added.db"; };' 2>&1 | sed 's/^/ns2 /' | cat_i
 $DIG +norec $DIGOPTS @10.53.0.2 mod.example ns > dig.out.ns2.1.$n || ret=1
 grep 'status: NOERROR' dig.out.ns2.1.$n > /dev/null || ret=1
-$RNDCCMD 10.53.0.2 modzone 'mod.example { type master; file "added.db"; allow-query { none; }; };' 2>&1 | sed 's/^/I:ns2 /'
+$RNDCCMD 10.53.0.2 modzone 'mod.example { type master; file "added.db"; allow-query { none; }; };' 2>&1 | sed 's/^/ns2 /' | cat_i
 $DIG +norec $DIGOPTS @10.53.0.2 mod.example ns > dig.out.ns2.2.$n || ret=1
 $RNDCCMD 10.53.0.2 showzone mod.example | grep 'allow-query { "none"; };' > /dev/null 2>&1 || ret=1
 n=`expr $n + 1`
@@ -368,11 +368,14 @@ status=`expr $status + $ret`
 echo_i "check that adding a 'master redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"." { type redirect; file "redirect.db"; };' > rndc.out.ns2.$n 2>&1 || ret=1
-$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || ret=1
-grep "type redirect;" showzone.out.ns2.$n > /dev/null || ret=1
-$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || ret=1
-grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || ret=1
-grep "serial: 0" zonestatus.out.ns2.$n > /dev/null || ret=1
+_check_add_master_redirect() (
+       $RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 &&
+       grep "type redirect;" showzone.out.ns2.$n > /dev/null &&
+       $RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 &&
+       grep "type: redirect" zonestatus.out.ns2.$n > /dev/null &&
+       grep "serial: 0" zonestatus.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_add_master_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -382,7 +385,7 @@ ret=0
 sleep 1
 cp -f ns2/redirect.db.2 ns2/redirect.db
 $RNDCCMD 10.53.0.2 reload -redirect > rndc.out.ns2.$n
-retry_quiet 5 check_zonestatus 2 || ret=1
+retry_quiet 10 check_zonestatus 2 || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -397,8 +400,11 @@ status=`expr $status + $ret`
 echo_i "check that deleting a 'master redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone -redirect > rndc.out.ns2.$n 2>&1 || ret=1
-$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1
-grep 'not found' showzone.out.ns2.$n > /dev/null || ret=1
+_check_deleting_master_redirect() (
+       $RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || true
+       grep 'not found' showzone.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_master_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -406,12 +412,14 @@ status=`expr $status + $ret`
 echo_i "check that adding a 'slave redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"." { type redirect; masters { 10.53.0.3;}; file "redirect.bk"; };' > rndc.out.ns2.$n 2>&1 || ret=1
-$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || ret=1
-grep "type redirect;" showzone.out.ns2.$n > /dev/null || ret=1
-sleep 1
-$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || ret=1
-grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || ret=1
-grep "serial: 0" zonestatus.out.ns2.$n > /dev/null || ret=1
+_check_adding_slave_redirect() (
+       $RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 &&
+       grep "type redirect;" showzone.out.ns2.$n > /dev/null &&
+       $RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 &&
+       grep "type: redirect" zonestatus.out.ns2.$n > /dev/null &&
+       grep "serial: 0" zonestatus.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_adding_slave_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -420,12 +428,13 @@ echo_i "check that retransfering a added 'slave redirect' zone works ($n)"
 ret=0
 cp -f ns3/redirect.db.2 ns3/redirect.db
 $RNDCCMD 10.53.0.3 reload . > showzone.out.ns3.$n 2>&1 || ret=1
-sleep 1
-$RNDCCMD 10.53.0.2 retransfer -redirect > rndc.out.ns2.$n 2>&1 || ret=1
-sleep 1
-$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || ret=1
-grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || ret=1
-grep "serial: 1" zonestatus.out.ns2.$n > /dev/null || ret=1
+_check_retransfering_slave_redirect() (
+       $RNDCCMD 10.53.0.2 retransfer -redirect > rndc.out.ns2.$n 2>&1 &&
+       $RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 &&
+       grep "type: redirect" zonestatus.out.ns2.$n > /dev/null &&
+       grep "serial: 1" zonestatus.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_retransfering_slave_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -433,8 +442,11 @@ status=`expr $status + $ret`
 echo_i "check that deleting a 'slave redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone -redirect > rndc.out.ns2.$n 2>&1 || ret=1
-$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1
-grep 'not found' showzone.out.ns2.$n > /dev/null || ret=1
+_check_deleting_slave_redirect() (
+       $RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || true
+       grep 'not found' showzone.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_slave_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -519,31 +531,30 @@ fi
 
 echo_i "checking rndc reload causes named to reload the external view's new zone config ($n)"
 ret=0
-$RNDCCMD 10.53.0.2 reload 2>&1 | sed 's/^/I:ns2 /'
-$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null || ret=1
-$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null || ret=1
+$RNDCCMD 10.53.0.2 reload 2>&1 | sed 's/^/ns2 /' | cat_i
+_check_rndc_reload_external_view_config() (
+       $DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n &&
+       grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null &&
+       $DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n &&
+       grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null &&
+       grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null
+)
+retry_quiet 10 _check_rndc_reload_external_view_config || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
 echo_i "checking rndc showzone with newly added zone ($n)"
-# loop because showzone may complain if zones are still being
-# loaded from the NZDB at this point.
-for try in 0 1 2 3 4 5; do
-    ret=0
-$RNDCCMD 10.53.0.2 showzone added.example in external > rndc.out.ns2.$n 2>/dev/null
-    if [ -z "$NZD" ]; then
-      expected='zone "added.example" in external { type master; file "added.db"; };'
-    else
-      expected='zone "added.example" { type master; file "added.db"; };'
-    fi
-    [ "`cat rndc.out.ns2.$n`" = "$expected" ] || ret=1
-    [ $ret -eq 0 ] && break
-    sleep 1
-done
+_check_rndc_showzone_newly_added() (
+       if [ -z "$NZD" ]; then
+               expected='zone "added.example" in external { type master; file "added.db"; };'
+       else
+               expected='zone "added.example" { type master; file "added.db"; };'
+       fi
+       $RNDCCMD 10.53.0.2 showzone added.example in external > rndc.out.ns2.$n 2>/dev/null &&
+       [ "`cat rndc.out.ns2.$n`" = "$expected" ]
+)
+retry_quiet 10  _check_rndc_showzone_newly_added || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -551,9 +562,12 @@ status=`expr $status + $ret`
 echo_i "deleting newly added zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone 'added.example in external' 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.$n || ret=1
-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.$n > /dev/null && ret=1
+_check_deleting_newly_added_zone() (
+       $DIG $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.$n &&
+       grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+       ! grep '^a.added.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_newly_added_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -583,11 +597,14 @@ status=`expr $status + $ret`
 echo_i "adding new zone again to external view ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone 'added.example in external { type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null || ret=1
-$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null || ret=1
+_check_adding_new_zone_again_external() (
+       $DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n &&
+       grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null &&
+       $DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n &&
+       grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null &&
+       grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null
+)
+retry_quiet 10 _check_adding_new_zone_again_external || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -692,20 +709,18 @@ then
     status=`expr $status + $ret`
 fi
 
+_check_version_bind() (
+       $DIG $DIGOPTS @10.53.0.3 version.bind txt ch > dig.out.test$n &&
+       grep "status: NOERROR" dig.out.test$n > /dev/null
+)
+
 echo_i "check that named restarts with multiple added zones ($n)"
 ret=0
 $RNDCCMD 10.53.0.3 addzone "test4.baz" '{ type master; file "e.db"; };' > /dev/null 2>&1 || ret=1
 $RNDCCMD 10.53.0.3 addzone "test5.baz" '{ type master; file "e.db"; };' > /dev/null 2>&1 || ret=1
 $PERL $SYSTEMTESTTOP/stop.pl addzone ns3
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} addzone ns3 || ret=1
-for try in 0 1 2 3 4 5 6 7 8 9; do
-    iret=0
-    $DIG $DIGOPTS @10.53.0.3 version.bind txt ch > dig.out.test$n || iret=1
-    grep "status: NOERROR" dig.out.test$n > /dev/null || iret=1
-    [ "$iret" -eq 0 ] && break
-    sleep 1
-done
-[ "$iret" -ne 0 ] && ret=1
+retry_quiet 10 _check_version_bind || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 n=`expr $n + 1`