tests: Add run-tests.py --shuffle-test

This optional argument can be used to randomize the order in which the
test cases are run. This can provide more coverage on testing
interactions of common use cases in various different sequences. Such
issues have already been found even with the fixed order of test cases,
but being able to reorder the tests makes this more efficient.

Signed-hostap: Jouni Malinen <j@w1.fi>

diff --git a/tests/hwsim/run-tests.py b/tests/hwsim/run-tests.py
index bd2d093e6ef4495d2ad7a3bb79438ada50cf4532..594c495afd0d2aab7b6aa304a83b8b328895cf7d 100755 (executable)
--- a/tests/hwsim/run-tests.py
+++ b/tests/hwsim/run-tests.py
@@ -121,6 +121,9 @@ def main():
                         help='collect tracing per test case (in log directory)')
     parser.add_argument('-D', action='store_true', dest='dmesg',
                         help='collect dmesg per test case (in log directory)')
+    parser.add_argument('--shuffle-tests', action='store_true',
+                        dest='shuffle_tests',
+                        help='Shuffle test cases to randomize order')
     parser.add_argument('-f', dest='testmodules', metavar='<test module>',
                         help='execute only tests from these test modules',
                         type=str, choices=[[]] + test_modules, nargs='+')
@@ -228,6 +231,10 @@ def main():
             name = t.__name__.replace('test_', '', 1)
             report(conn, False, args.build, args.commit, run, name, 'NOTRUN', 0)
 
+    if args.shuffle_tests:
+        from random import shuffle
+        shuffle(tests_to_run)
+
     for t in tests_to_run:
         name = t.__name__.replace('test_', '', 1)
         if log_handler: