tests: Write test case result summary into a file

This can be used to build statistics from test execution without having
to go through number of log files.

Signed-hostap: Jouni Malinen <j@w1.fi>

diff --git a/tests/hwsim/run-all.sh b/tests/hwsim/run-all.sh
index dbd74034a20675ed4a9ba36c335bfdf000d09baf..bb13d514c8e0fd27275225b383bb06a63170ff69 100755 (executable)
--- a/tests/hwsim/run-all.sh
+++ b/tests/hwsim/run-all.sh
@@ -11,7 +11,7 @@ if [ "x$1" = "xconcurrent-valgrind" ]; then
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
     rm logs/last-debug
     for i in autogo discovery grpform; do
-       ./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -f test_p2p_$i.py || errors=1
+       ./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
        cat logs/$DATE-run-$i >> logs/last-debug
     done
     ./stop-wifi.sh
@@ -32,7 +32,7 @@ elif [ "x$1" = "xconcurrent" ]; then
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
     rm logs/last-debug
     for i in autogo discovery grpform; do
-       ./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -f test_p2p_$i.py || errors=1
+       ./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
        cat logs/$DATE-run-$i >> logs/last-debug
     done
     ./stop-wifi.sh
@@ -46,7 +46,7 @@ elif [ "x$1" = "xvalgrind" ]; then
        exit 1
     fi
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
-    ./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed || errors=1
+    ./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt || errors=1
     cat logs/$DATE-run > logs/last-debug
     ./stop-wifi.sh
     failures=`grep "ERROR SUMMARY" logs/$DATE-valgrind-* | grep -v " 0 errors" | wc -l`
@@ -64,7 +64,7 @@ elif [ "x$1" = "xtrace" ]; then
        exit 1
     fi
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
-    sudo trace-cmd record -o logs/$DATE-trace.dat -e mac80211 -e cfg80211 su $USER -c "./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed" || errors=1
+    sudo trace-cmd record -o logs/$DATE-trace.dat -e mac80211 -e cfg80211 su $USER -c "./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt" || errors=1
     if [ -e logs/$DATE-failed ]; then
        error=1
     fi
@@ -81,7 +81,7 @@ else
        exit 1
     fi
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
-    ./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed || errors=1
+    ./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt || errors=1
     cat logs/$DATE-run > logs/last-debug
     ./stop-wifi.sh
     if [ $errors -gt 0 ]; then

diff --git a/tests/hwsim/run-tests.py b/tests/hwsim/run-tests.py
index 14de7db3321b117d739e0d53c726d35b482a57e7..e81c487be86f71d5c53fa889ae4c60d7d0911901 100755 (executable)
--- a/tests/hwsim/run-tests.py
+++ b/tests/hwsim/run-tests.py
@@ -33,6 +33,7 @@ def main():
     test_file = None
     error_file = None
     log_file = None
+    results_file = None
     idx = 1
     if len(sys.argv) > 1 and sys.argv[1] == '-d':
         logging.basicConfig(level=logging.DEBUG)
@@ -51,6 +52,10 @@ def main():
         error_file = sys.argv[idx + 1]
         idx = idx + 2
 
+    if len(sys.argv) > idx + 1 and sys.argv[idx] == '-r':
+        results_file = sys.argv[idx + 1]
+        idx = idx + 2
+
     if len(sys.argv) > idx + 1 and sys.argv[idx] == '-f':
         test_file = sys.argv[idx + 1]
         idx = idx + 2
@@ -117,17 +122,27 @@ def main():
             passed.append(t.__name__)
             end = datetime.now()
             diff = end - start
-            logger.info("PASS " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end))
+            result = "PASS " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
+            logger.info(result)
             if log_file:
-                print "PASS " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
+                print result
+            if results_file:
+                f = open(results_file, 'a')
+                f.write(result + "\n")
+                f.close()
         except Exception, e:
             end = datetime.now()
             diff = end - start
             logger.info(e)
             failed.append(t.__name__)
-            logger.info("FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end))
+            result = "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
+            logger.info(result)
             if log_file:
-                print "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
+                print result
+            if results_file:
+                f = open(results_file, 'a')
+                f.write(result + "\n")
+                f.close()
         for d in dev:
             try:
                 d.request("NOTE TEST-STOP " + t.__name__)