From: Jason Ish <ish@unx.ca>
Date: Tue, 16 Jan 2018 20:28:02 +0000 (-0600)
Subject: test.yaml: count field to execute test X number of times
X-Git-Tag: suricata-6.0.4~532
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a7bc8cf7d9f8537b3f07376e17feaaac105c2ade;p=thirdparty%2Fsuricata-verify.git

test.yaml: count field to execute test X number of times
---

diff --git a/README.md b/README.md
index b79d3cb40..020c0a078 100644
--- a/README.md
+++ b/README.md
@@ -76,3 +76,7 @@ args:
 command: |
   ${SRCDIR}/src/suricata -T -c ${TEST_DIR}/suricata.yaml -vvv \
       -l ${TEST_DIR}/output --set default-rule-path="${TEST_DIR}"
+
+# Execute Suricata with the test parameters this many times. All checks will
+# done after each iteration.
+count: 10
diff --git a/run.py b/run.py
index bca78450f..1addce7f6 100755
--- a/run.py
+++ b/run.py
@@ -322,26 +322,37 @@ class TestRunner:
         stdout = open(os.path.join(self.output, "stdout"), "w")
         stderr = open(os.path.join(self.output, "stderr"), "w")
 
-        open(os.path.join(self.output, "cmdline"), "w").write(
-            " ".join(args))
+        if "count" in self.config:
+            count = self.config["count"]
+        else:
+            count = 1
 
-        p = subprocess.Popen(
-            args, shell=shell, cwd=self.directory, env=env,
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        for _ in range(count):
 
-        self.start_reader(p.stdout, stdout)
-        self.start_reader(p.stderr, stderr)
+            open(os.path.join(self.output, "cmdline"), "w").write(
+                " ".join(args))
 
-        for r in self.readers:
-            r.join()
+            p = subprocess.Popen(
+                args, shell=shell, cwd=self.directory, env=env,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
-        r = p.wait()
+            self.start_reader(p.stdout, stdout)
+            self.start_reader(p.stderr, stderr)
 
-        if r != 0:
-            print("FAIL: process returned with non-0 exit code: %d" % r)
-            return False
+            for r in self.readers:
+                r.join()
 
-        return self.check()
+            r = p.wait()
+
+            if r != 0:
+                print("FAIL: process returned with non-0 exit code: %d" % r)
+                return False
+
+            if not self.check():
+                return False
+
+        print("OK%s" % (" (%dx)" % count if count > 1 else ""))
+        return True
 
     def check(self):
 
@@ -369,13 +380,11 @@ class TestRunner:
             os.chdir(pdir)
 
         if not os.path.exists(os.path.join(self.directory, "check.sh")):
-            print("OK")
             return True
         r = subprocess.call(["./check.sh"], cwd=self.directory)
         if r != 0:
             print("FAILED: verification failed")
             return False
-        print("OK")
         return True
         
     def default_args(self):