Store system test output in bin/tests/system/

Some clean.sh scripts contain overly broad file deletion wildcards which
cause the test.output file (used by the system test framework for
collecting output) in a given system test's directory to be erroneously
removed immediately after the test is started (due to setup.sh scripts
calling clean.sh at the beginning).  This prevents the test's output
from being placed in bin/tests/system/systests.output at the end of a
test suite run and thus can lead to test failures being ignored.  Fix by
storing each test's output in a test.output.<test-name> file in
bin/tests/system/, which prevents clean.sh scripts from removing it (as
they should only ever affect files contained in a given system test's
directory).

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index c85b5b92da3f25b30d419f074e8094437b8df737..f51188bc289a988aedf1186b81d45c6d05d6ee89 100644 (file)
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -5,7 +5,7 @@ named.lock
 named.pid
 named.run
 /feature-test
-**/test.output
+/test.output.*
 /systests.output
 /random.data
 parallel.mk

diff --git a/bin/tests/system/README b/bin/tests/system/README
index 13286c054e6815b4c24157897c79eba21be50a6b..552344ba87af44f00887f2323c31828511403662 100644 (file)
--- a/bin/tests/system/README
+++ b/bin/tests/system/README
@@ -717,17 +717,17 @@ separated by at least 100 ports.
 
 Cleaning Up From Tests
 ---
-When a test is run, files are created in the test directory.  These files fall
-into three categories:
+When a test is run, up to three different types of files are created:
 
-1. Files generated by the test itself, e.g. output from "dig" and "rndc".
+1. Files generated by the test itself, e.g. output from "dig" and "rndc", are
+stored in the test directory.
 
 2. Files produced by named which may not be cleaned up if named exits
-abnormally, e.g. core files, PID files etc.
+abnormally, e.g. core files, PID files etc., are stored in the test directory.
 
-3. The file "test.output" containing the text written to stdout by the test.
-This is only produced when the test is run as part of the entire test suite
-(e.g. via "runall.sh").
+3. A file "test.output.<test-name>" containing the text written to stdout by the
+test is written to bin/tests/system/.  This file is only produced when the test
+is run as part of the entire test suite (e.g. via "runall.sh").
 
 If the test fails, all these files are retained.  But if the test succeeds,
 they are cleaned up at different times:
@@ -739,9 +739,9 @@ called from "run.sh".
 2. Files that may not be cleaned up if named exits abnormally are removed
 by the system's "clean.sh".
 
-3. "test.output" files are deleted when the test suite ends.  At this point,
-the file "testsummary.sh" is called which concatenates all the "test.output"
-files into a single "systest.output" file before deleting them.
+3. "test.output.*" files are deleted when the test suite ends.  At this point,
+the file "testsummary.sh" is called which concatenates all the "test.output.*"
+files into a single "systests.output" file before deleting them.
 
 A complication arises with the "test.output" file however:
 

diff --git a/bin/tests/system/cleanall.sh b/bin/tests/system/cleanall.sh
index d57d450bf2130bcf8c88f8439d6671c88a372ecd..d4b3c665cc618c4238190def61fc6f0a304444e4 100644 (file)
--- a/bin/tests/system/cleanall.sh
+++ b/bin/tests/system/cleanall.sh
@@ -30,6 +30,6 @@ rm -f $SYSTEMTESTTOP/random.data
 for d in $SUBDIRS
 do
    test ! -f $d/clean.sh || ( cd $d && $SHELL clean.sh )
-   test -f $d/test.output && rm $d/test.output
+   rm -f test.output.$d
    test -d $d && find $d -type d -exec rmdir '{}' \; 2> /dev/null
 done

diff --git a/bin/tests/system/parallel.sh b/bin/tests/system/parallel.sh
index 24b465e105acf53e55e5e500d22f6ef108ba49ed..2eab88c444ee667b386fe70ae375816df811c283 100644 (file)
--- a/bin/tests/system/parallel.sh
+++ b/bin/tests/system/parallel.sh
@@ -29,6 +29,6 @@ port=${STARTPORT:-5000}
 for directory in $PARALLELDIRS ; do
         echo
         echo "test-`echo $directory | tr _ -`: check_interfaces"
-        echo " @${SHELL} ./run.sh -r -p $port $directory 2>&1 | tee $directory/test.output"
+        echo " @${SHELL} ./run.sh -r -p $port $directory 2>&1 | tee test.output.$directory"
         port=`expr $port + 100`
 done

diff --git a/bin/tests/system/runsequential.sh b/bin/tests/system/runsequential.sh
index 5e352e9bc5a581731e800f99b73513f18c718416..6380f4c2f225487391f866179795366305de8453 100755 (executable)
--- a/bin/tests/system/runsequential.sh
+++ b/bin/tests/system/runsequential.sh
@@ -21,5 +21,5 @@ SYSTEMTESTTOP=.
 
 for d in $SEQUENTIALDIRS
 do
-    $SHELL run.sh "${@}" $d 2>&1 | tee $d/test.output
+    $SHELL run.sh "${@}" $d 2>&1 | tee test.output.$d
 done

diff --git a/bin/tests/system/testsummary.sh b/bin/tests/system/testsummary.sh
index 1c008cdbad0a8605624d3c96d70d7c29b045fa4d..a6f2bc03f9105d406b5e5ba92f5e78d924eb8c03 100644 (file)
--- a/bin/tests/system/testsummary.sh
+++ b/bin/tests/system/testsummary.sh
@@ -9,7 +9,7 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
-# Creates the system tests output file from the various test.output files.  It
+# Creates the system tests output file from the various test.output.* files.  It
 # then searches that file and prints the number of tests passed, failed, not
 # run.  It also checks whether the IP addresses 10.53.0.[1-8] were set up and,
 # if not, prints a warning.
@@ -17,7 +17,7 @@
 # Usage:
 #    testsummary.sh [-n]
 #
-# -n   Do NOT delete the individual test.output files after concatenating
+# -n   Do NOT delete the individual test.output.* files after concatenating
 #      them into systests.output.
 #
 # Status return:
@@ -35,13 +35,13 @@ while getopts "n" flag; do
     esac
 done
 
-if [ `ls */test.output 2> /dev/null | wc -l` -eq 0 ]; then
-    echowarn "I:No 'test.output' files were found."
+if [ `ls test.output.* 2> /dev/null | wc -l` -eq 0 ]; then
+    echowarn "I:No 'test.output.*' files were found."
     echowarn "I:Printing summary from pre-existing 'systests.output'."
 else
-    cat */test.output > systests.output
+    cat test.output.* > systests.output
     if [ $keepfile -eq 0 ]; then
-        rm -f */test.output
+        rm -f test.output.*
     fi
 fi