From: Michał Kępień <michal@isc.org>
Date: Tue, 18 Jun 2019 07:14:07 +0000 (+0200)
Subject: Always fail a system test if crashes are detected
X-Git-Tag: v9.15.2~49^2
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=7706f22924dac215e22d4be4322f89814041a3a8;p=thirdparty%2Fbind9.git

Always fail a system test if crashes are detected

In certain situations (e.g. a named instance crashing upon shutdown in a
system test which involves shutting down a server and restarting it
afterwards), a system test may succeed despite a named crash being
triggered.  This must never be the case.  Extend run.sh to mark a test
as failed if core dumps or log lines indicating assertion failures are
detected (the latter is only an extra measure aimed at test environments
in which core dumps are not generated; note that some types of crashes,
e.g. segmentation faults, will not be detected using this method alone).
---

diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index 6754a4252d0..4344b4412e5 100755
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -188,19 +188,30 @@ status=`expr $status + $?`
 if [ $status != 0 ]; then
     echofail "R:$systest:FAIL"
     # Do not clean up - we need the evidence.
-    find . -name core -exec chmod 0644 '{}' \;
 else
-    echopass "R:$systest:PASS"
-    if $clean
-    then
-	$SHELL clean.sh $runall $systest "$@"
-	if test -d ../../../.git
-	then
-	    git status -su --ignored $systest 2>/dev/null | \
-	    sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \
-	    -e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \
-	    -e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p'
-	fi
+    core_dumps="$(find $systest/ -name 'core*' | sort | tr '\n' ' ')"
+    assertion_failures=$(find $systest/ -name named.run | xargs grep "assertion failure" | wc -l)
+    if [ -n "$core_dumps" ]; then
+        echoinfo "I:$systest:Test claims success despite crashes: $core_dumps"
+        echofail "R:$systest:FAIL"
+        # Do not clean up - we need the evidence.
+    elif [ $assertion_failures -ne 0 ]; then
+        echoinfo "I:$systest:Test claims success despite $assertion_failures assertion failure(s)"
+        echofail "R:$systest:FAIL"
+        # Do not clean up - we need the evidence.
+    else
+        echopass "R:$systest:PASS"
+        if $clean
+        then
+            $SHELL clean.sh $runall $systest "$@"
+            if test -d ../../../.git
+            then
+                git status -su --ignored $systest 2>/dev/null | \
+                sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \
+                -e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \
+                -e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p'
+            fi
+        fi
     fi
 fi