From 38a4bedfcd84b7f2704fd8dbc034f5339700fcbd Mon Sep 17 00:00:00 2001 From: =?utf8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Fri, 6 Dec 2019 14:11:01 +0100 Subject: [PATCH] Store system test output in bin/tests/system/ Some clean.sh scripts contain overly broad file deletion wildcards which cause the test.output file (used by the system test framework for collecting output) in a given system test's directory to be erroneously removed immediately after the test is started (due to setup.sh scripts calling clean.sh at the beginning). This prevents the test's output from being placed in bin/tests/system/systests.output at the end of a test suite run and thus can lead to test failures being ignored. Fix by storing each test's output in a test.output. file in bin/tests/system/, which prevents clean.sh scripts from removing it (as they should only ever affect files contained in a given system test's directory). (cherry picked from commit b0916bba41b2898185c91a9ddc07dea42d1bdfaa) --- bin/tests/system/.gitignore | 2 +- bin/tests/system/README | 20 ++++++++++---------- bin/tests/system/cleanall.sh | 2 +- bin/tests/system/parallel.sh | 2 +- bin/tests/system/runsequential.sh | 2 +- bin/tests/system/testsummary.sh | 12 ++++++------ 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index c85b5b92da3..f51188bc289 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -5,7 +5,7 @@ named.lock named.pid named.run /feature-test -**/test.output +/test.output.* /systests.output /random.data parallel.mk diff --git a/bin/tests/system/README b/bin/tests/system/README index f527e790d41..6b11a401bb0 100644 --- a/bin/tests/system/README +++ b/bin/tests/system/README @@ -722,17 +722,17 @@ separated by at least 100 ports. Cleaning Up From Tests --- -When a test is run, files are created in the test directory. These files fall -into three categories: +When a test is run, up to three different types of files are created: -1. Files generated by the test itself, e.g. output from "dig" and "rndc". +1. Files generated by the test itself, e.g. output from "dig" and "rndc", are +stored in the test directory. 2. Files produced by named which may not be cleaned up if named exits -abnormally, e.g. core files, PID files etc. +abnormally, e.g. core files, PID files etc., are stored in the test directory. -3. The file "test.output" containing the text written to stdout by the test. -This is only produced when the test is run as part of the entire test suite -(e.g. via "runall.sh"). +3. A file "test.output." containing the text written to stdout by the +test is written to bin/tests/system/. This file is only produced when the test +is run as part of the entire test suite (e.g. via "runall.sh"). If the test fails, all these files are retained. But if the test succeeds, they are cleaned up at different times: @@ -744,9 +744,9 @@ called from "run.sh". 2. Files that may not be cleaned up if named exits abnormally are removed by the system's "clean.sh". -3. "test.output" files are deleted when the test suite ends. At this point, -the file "testsummary.sh" is called which concatenates all the "test.output" -files into a single "systest.output" file before deleting them. +3. "test.output.*" files are deleted when the test suite ends. At this point, +the file "testsummary.sh" is called which concatenates all the "test.output.*" +files into a single "systests.output" file before deleting them. A complication arises with the "test.output" file however: diff --git a/bin/tests/system/cleanall.sh b/bin/tests/system/cleanall.sh index d57d450bf21..d4b3c665cc6 100644 --- a/bin/tests/system/cleanall.sh +++ b/bin/tests/system/cleanall.sh @@ -30,6 +30,6 @@ rm -f $SYSTEMTESTTOP/random.data for d in $SUBDIRS do test ! -f $d/clean.sh || ( cd $d && $SHELL clean.sh ) - test -f $d/test.output && rm $d/test.output + rm -f test.output.$d test -d $d && find $d -type d -exec rmdir '{}' \; 2> /dev/null done diff --git a/bin/tests/system/parallel.sh b/bin/tests/system/parallel.sh index 24b465e105a..2eab88c444e 100644 --- a/bin/tests/system/parallel.sh +++ b/bin/tests/system/parallel.sh @@ -29,6 +29,6 @@ port=${STARTPORT:-5000} for directory in $PARALLELDIRS ; do echo echo "test-`echo $directory | tr _ -`: check_interfaces" - echo " @${SHELL} ./run.sh -r -p $port $directory 2>&1 | tee $directory/test.output" + echo " @${SHELL} ./run.sh -r -p $port $directory 2>&1 | tee test.output.$directory" port=`expr $port + 100` done diff --git a/bin/tests/system/runsequential.sh b/bin/tests/system/runsequential.sh index 5e352e9bc5a..6380f4c2f22 100755 --- a/bin/tests/system/runsequential.sh +++ b/bin/tests/system/runsequential.sh @@ -21,5 +21,5 @@ SYSTEMTESTTOP=. for d in $SEQUENTIALDIRS do - $SHELL run.sh "${@}" $d 2>&1 | tee $d/test.output + $SHELL run.sh "${@}" $d 2>&1 | tee test.output.$d done diff --git a/bin/tests/system/testsummary.sh b/bin/tests/system/testsummary.sh index 1c008cdbad0..a6f2bc03f91 100644 --- a/bin/tests/system/testsummary.sh +++ b/bin/tests/system/testsummary.sh @@ -9,7 +9,7 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. -# Creates the system tests output file from the various test.output files. It +# Creates the system tests output file from the various test.output.* files. It # then searches that file and prints the number of tests passed, failed, not # run. It also checks whether the IP addresses 10.53.0.[1-8] were set up and, # if not, prints a warning. @@ -17,7 +17,7 @@ # Usage: # testsummary.sh [-n] # -# -n Do NOT delete the individual test.output files after concatenating +# -n Do NOT delete the individual test.output.* files after concatenating # them into systests.output. # # Status return: @@ -35,13 +35,13 @@ while getopts "n" flag; do esac done -if [ `ls */test.output 2> /dev/null | wc -l` -eq 0 ]; then - echowarn "I:No 'test.output' files were found." +if [ `ls test.output.* 2> /dev/null | wc -l` -eq 0 ]; then + echowarn "I:No 'test.output.*' files were found." echowarn "I:Printing summary from pre-existing 'systests.output'." else - cat */test.output > systests.output + cat test.output.* > systests.output if [ $keepfile -eq 0 ]; then - rm -f */test.output + rm -f test.output.* fi fi -- 2.47.3