hwsim tests: Prefill database in VM tests

In some cases, e.g., with the VM tests if the VM crashes, it
can be useful to know which tests should have run but didn't
(or didn't finish). In order to catch these more easily, add
an option to prefill the database with all tests at the very
beginning of the testing (in a new NOTRUN state) and use the
option in the VM tests.

Signed-hostap: Johannes Berg <johannes.berg@intel.com>

diff --git a/tests/hwsim/run-all.sh b/tests/hwsim/run-all.sh
index 84b5baefe94c2264c804c35d78ff55b4f228e6ae..31794463bd182ee343b7a0148ed9a8f9575f2df3 100755 (executable)
--- a/tests/hwsim/run-all.sh
+++ b/tests/hwsim/run-all.sh
@@ -19,6 +19,9 @@ else
     if [ -n "$BUILD" ]; then
        DB="$DB -b $BUILD"
     fi
+    if [ "$PREFILL_DB" = "y" ] ; then
+        DB="$DB --prefill-tests"
+    fi
 fi
 
 if [ "x$1" = "xconcurrent-valgrind" ]; then

diff --git a/tests/hwsim/run-tests.py b/tests/hwsim/run-tests.py
index b486013e82f30da47856bfe7170397051af8bdbe..bd2d093e6ef4495d2ad7a3bb79438ada50cf4532 100755 (executable)
--- a/tests/hwsim/run-tests.py
+++ b/tests/hwsim/run-tests.py
@@ -35,14 +35,16 @@ def reset_devs(dev, apdev):
     for ap in apdev:
         hapd.remove(ap['ifname'])
 
-def report(conn, build, commit, run, test, result, diff):
+def report(conn, prefill, build, commit, run, test, result, duration):
     if conn:
         if not build:
             build = ''
         if not commit:
             commit = ''
+        if prefill:
+            conn.execute('DELETE FROM results WHERE test=? AND run=? AND result=?', (test, run, 'NOTRUN'))
         sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES(?, ?, ?, ?, ?, ?, ?)"
-        params = (test, result, run, time.time(), diff.total_seconds(), build, commit)
+        params = (test, result, run, time.time(), duration, build, commit)
         try:
             conn.execute(sql, params)
             conn.commit()
@@ -108,6 +110,8 @@ def main():
 
     parser.add_argument('-S', metavar='<sqlite3 db>', dest='database',
                         help='database to write results to')
+    parser.add_argument('--prefill-tests', action='store_true', dest='prefill',
+                        help='prefill test database with NOTRUN before all tests')
     parser.add_argument('--commit', metavar='<commit id>',
                         help='commit ID, only for database')
     parser.add_argument('-b', metavar='<build>', dest='build', help='build ID')
@@ -208,6 +212,7 @@ def main():
     if args.dmesg:
         subprocess.call(['sudo', 'dmesg', '-c'], stdout=open('/dev/null', 'w'))
 
+    tests_to_run = []
     for t in tests:
         name = t.__name__.replace('test_', '', 1)
         if args.tests:
@@ -216,7 +221,15 @@ def main():
         if args.testmodules:
             if not t.__module__.replace('test_', '', 1) in args.testmodules:
                 continue
+        tests_to_run.append(t)
 
+    if conn and args.prefill:
+        for t in tests_to_run:
+            name = t.__name__.replace('test_', '', 1)
+            report(conn, False, args.build, args.commit, run, name, 'NOTRUN', 0)
+
+    for t in tests_to_run:
+        name = t.__name__.replace('test_', '', 1)
         if log_handler:
             log_handler.stream.close()
             logger.removeHandler(log_handler)
@@ -286,7 +299,7 @@ def main():
 
         end = datetime.now()
         diff = end - start
-        report(conn, args.build, args.commit, run, name, result, diff)
+        report(conn, args.prefill, args.build, args.commit, run, name, result, diff.total_seconds())
         result = result + " " + name + " "
         result = result + str(diff.total_seconds()) + " " + str(end)
         logger.info(result)

diff --git a/tests/hwsim/vm/inside.sh b/tests/hwsim/vm/inside.sh
index cbef727948b7b30716997ca28ad0afd121e89029..d14bf01b51cb97392fc2bd4f036a941dc89b054d 100755 (executable)
--- a/tests/hwsim/vm/inside.sh
+++ b/tests/hwsim/vm/inside.sh
@@ -64,6 +64,7 @@ else
        export USER=0
        export LOGDIR=/tmp/logs
        export DBFILE=$LOGDIR/results.db
+       export PREFILL_DB=y
 
        cd $TESTDIR
        ./run-all.sh $ARGS >/dev/ttyS0 2>&1