runner: sort the tests alphabetically then run

diff --git a/run.py b/run.py
index 1b0fd3739a57b28e3532efa88ba29e382b752959..958bd3b9abb3570eb21447fda5e1adcca1970359 100755 (executable)
--- a/run.py
+++ b/run.py
@@ -496,8 +496,9 @@ def main():
     if args.dir:
         tdir = os.path.abspath(args.dir)
 
+    # First gather the tests so we can run them in alphabetic order.
+    tests = []
     for dirpath, dirnames, filenames in os.walk(tdir):
-
         # The top directory is not a test...
         if dirpath == os.path.join(topdir, "tests"):
             continue
@@ -507,38 +508,38 @@ def main():
         # We only want to go one level deep.
         dirnames[0:] = []
 
-        name = os.path.basename(dirpath)
-
-        do_test = False
         if not args.patterns:
-            do_test = True
+            tests.append(dirpath)
         else:
-            # If a test matches a pattern, we do not skip it.
             for pattern in args.patterns:
-                if name.find(pattern) > -1:
-                    do_test = True
-                    break
-
-        if do_test:
-            test_runner = TestRunner(
-                cwd, dirpath, suricata_config, args.verbose)
-            try:
-                if test_runner.run():
-                    passed += 1
-                else:
-                    failed += 1
-                    if args.fail:
-                        return 1
-            except UnsatisfiedRequirementError as err:
-                print("SKIPPED: %s" % (str(err)))
-                skipped += 1
-            except TestError as err:
-                print("FAIL: %s" % (str(err)))
+                if os.path.basename(dirpath).find(pattern) > -1:
+                    tests.append(dirpath)
+
+    # Sort alphabetically.
+    tests.sort()
+
+    for dirpath in tests:
+        name = os.path.basename(dirpath)
+
+        test_runner = TestRunner(
+            cwd, dirpath, suricata_config, args.verbose)
+        try:
+            if test_runner.run():
+                passed += 1
+            else:
                 failed += 1
                 if args.fail:
                     return 1
-            except Exception as err:
-                raise
+        except UnsatisfiedRequirementError as err:
+            print("SKIPPED: %s" % (str(err)))
+            skipped += 1
+        except TestError as err:
+            print("FAIL: %s" % (str(err)))
+            failed += 1
+            if args.fail:
+                return 1
+        except Exception as err:
+            raise
 
     print("")
     print("PASSED:  %d" % (passed))