Modify custom-test-driver to interpret JUnit results

Pytest provides JUnit output and uses different exit codes from
Automake. Use the conversion script to interpret the JUnit test results
from python rather than relying on the status code.

(cherry picked from commit 295890a16b24f740b8bc8b7c75a4ded9802cb3ff)

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index a54b7c9565bbf47675d85d40c81d3524f32eb5ee..f946793ae7b9545257c4edf8a35798dbf2c20964 100644 (file)
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -16,6 +16,7 @@ named.run
 parallel.mk
 /*.log
 /*.trs
+/*.xml
 /resolve
 /legacy.run.sh
 /run.log

diff --git a/bin/tests/system/custom-test-driver b/bin/tests/system/custom-test-driver
index 7499aa0bc036bb7d50e343182b9678ce04849192..fed98cccc6498ac9c96b613f25e3314fef4ec5f5 100755 (executable)
--- a/bin/tests/system/custom-test-driver
+++ b/bin/tests/system/custom-test-driver
@@ -56,7 +56,7 @@ END
 test_name= # Used for reporting.
 log_file=  # Where to save the output of the test script.
 trs_file=  # Where to save the metadata of the test run.
-status_file= # Where to save the status of the test run.
+junit_file=  # Where to save pytest junit output.
 expect_failure=no
 color_tests=no
 enable_hard_errors=yes
@@ -67,8 +67,7 @@ while test $# -gt 0; do
   --version) echo "test-driver $scriptversion"; exit $?;;
   --test-name) test_name=$2; shift;;
   --log-file) log_file=$2; shift;;
-  --trs-file) trs_file=$2; shift;;
-  --status-file) status_file=$2; shift;;
+  --trs-file) trs_file=$2; junit_file=$(echo $trs_file | sed 's/\.trs$/\.xml/'); shift;;
   --color-tests) color_tests=$2; shift;;
   --expect-failure) expect_failure=$2; shift;;
   --enable-hard-errors) enable_hard_errors=$2; shift;;
@@ -104,22 +103,22 @@ else
   red= grn= lgn= blu= mgn= std=
 fi
 
-do_exit='rm -f $log_file $trs_file $status_file; (exit $st); exit $st'
+do_exit='rm -f $log_file $trs_file $junit_file; (exit $st); exit $st'
 trap "st=129; $do_exit" 1
 trap "st=130; $do_exit" 2
 trap "st=141; $do_exit" 13
 trap "st=143; $do_exit" 15
 
-# Set default
-test x"$status_file" = x && status_file=$(mktemp ./custom-test-runner.XXXXXX)
 # Test script is run here.
 if test $verbose = yes; then
-  ("$@" 2>&1; echo $? > "$status_file") | tee $log_file
+  "$@" --junit-xml $PWD/$junit_file 2>&1 | tee $log_file
 else
-  "$@" >$log_file 2>&1; echo $? > "$status_file"
+  "$@" --junit-xml $PWD/$junit_file >$log_file 2>&1
 fi
-read -r estatus < "$status_file"
-rm "$status_file"
+
+# Run junit to trs converter script.
+./convert-junit-to-trs.py $junit_file > $trs_file
+estatus=$?
 
 if test $enable_hard_errors = no && test $estatus -eq 99; then
   tweaked_estatus=1
@@ -145,8 +144,7 @@ echo "$res $test_name (exit status: $estatus)" >>$log_file
 # Report outcome to console.
 echo "${col}${res}${std}: $test_name"
 
-# Register the test result, and other relevant metadata.
-echo ":test-result: $res" > $trs_file
+# Register other relevant test metadata.
 echo ":global-test-result: $res" >> $trs_file
 echo ":recheck: $recheck" >> $trs_file
 echo ":copy-in-global-log: $gcopy" >> $trs_file