test.yaml: count field to execute test X number of times

diff --git a/README.md b/README.md
index b79d3cb40246c50ddd64b2c210a7a4e639445bcf..020c0a078bea07f16df968d33210066733f4ae95 100644 (file)
--- a/README.md
+++ b/README.md
@@ -76,3 +76,7 @@ args:
 command: |
   ${SRCDIR}/src/suricata -T -c ${TEST_DIR}/suricata.yaml -vvv \
       -l ${TEST_DIR}/output --set default-rule-path="${TEST_DIR}"
+
+# Execute Suricata with the test parameters this many times. All checks will
+# done after each iteration.
+count: 10

diff --git a/run.py b/run.py
index bca78450f42b917c8bb7bccd7e31e762d6f2ddee..1addce7f6ad073a9aa4fbec507bc5fb911183eb5 100755 (executable)
--- a/run.py
+++ b/run.py
@@ -322,26 +322,37 @@ class TestRunner:
         stdout = open(os.path.join(self.output, "stdout"), "w")
         stderr = open(os.path.join(self.output, "stderr"), "w")
 
-        open(os.path.join(self.output, "cmdline"), "w").write(
-            " ".join(args))
+        if "count" in self.config:
+            count = self.config["count"]
+        else:
+            count = 1
 
-        p = subprocess.Popen(
-            args, shell=shell, cwd=self.directory, env=env,
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        for _ in range(count):
 
-        self.start_reader(p.stdout, stdout)
-        self.start_reader(p.stderr, stderr)
+            open(os.path.join(self.output, "cmdline"), "w").write(
+                " ".join(args))
 
-        for r in self.readers:
-            r.join()
+            p = subprocess.Popen(
+                args, shell=shell, cwd=self.directory, env=env,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
-        r = p.wait()
+            self.start_reader(p.stdout, stdout)
+            self.start_reader(p.stderr, stderr)
 
-        if r != 0:
-            print("FAIL: process returned with non-0 exit code: %d" % r)
-            return False
+            for r in self.readers:
+                r.join()
 
-        return self.check()
+            r = p.wait()
+
+            if r != 0:
+                print("FAIL: process returned with non-0 exit code: %d" % r)
+                return False
+
+            if not self.check():
+                return False
+
+        print("OK%s" % (" (%dx)" % count if count > 1 else ""))
+        return True
 
     def check(self):
 
@@ -369,13 +380,11 @@ class TestRunner:
             os.chdir(pdir)
 
         if not os.path.exists(os.path.join(self.directory, "check.sh")):
-            print("OK")
             return True
         r = subprocess.call(["./check.sh"], cwd=self.directory)
         if r != 0:
             print("FAILED: verification failed")
             return False
-        print("OK")
         return True
         
     def default_args(self):