runner: error out on errors from the runner

Currently exceptions from tests that are not explicitly handled are lost
due to the way Python's multiprocessing module works.  This means that
programming errors in the runner are silently ignore with the test not
being run or counted.

Instead, log the traceback for any unexpected exception and terminate
the runner, as this is programming error in the runner and should always
be fatal.

diff --git a/run.py b/run.py
index 2126b03fb079ceb6e336c2e79374b03df6e69e5c..2a039b1bf4028765d6e390ea1c01efe946cacc1a 100755 (executable)
--- a/run.py
+++ b/run.py
@@ -43,6 +43,7 @@ import threading
 import filecmp
 import subprocess
 import yaml
+import traceback
 
 VALIDATE_EVE = False
 WIN32 = sys.platform == "win32"
@@ -896,6 +897,15 @@ def run_test(dirpath, args, cwd, suricata_config):
         check_args_fail()
         with lock:
             count_dict["failed"] += 1
+    except Exception as err:
+        print("===> {}: FAILED: Unexpected exception: {}".format(os.path.basename(dirpath), err))
+        traceback.print_exc()
+
+        # Always terminate the runner on this type of error, as its an error in the framework.
+        with lock:
+            check_args['fail'] = 1
+            count_dict["failed"] += 1
+            raise TerminatePoolError()
 
 def run_mp(jobs, tests, dirpath, args, cwd, suricata_config):
     print("Number of concurrent jobs: %d" % jobs)