Ensure pytest runner get proper outcome from flaky reruns

When a test is re-run by the flaky plugin, the TestReport outcomes
collected in the pytest_runtest_makereport() hook should be overriden.
Each of the setup/call/teardown phases is reported again and since we
care about the overall outcome, their respective results should be
overriden so that only the outcome from the final test (re)run gets
reported.

Prior to this change, it lead to a situation where an extra_artifact
generated during the test might be ignored. This was caused because the
check was skipped, since the test was incorrectly considered as "failed"
in the case where the test would fail on the first run, but pass on a
subsequent flaky rerun.

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index e5cbd3f326351e116f63c9ea5bc32ab17dd6442a..919a188a96966f9eb1a2d1abd1da090474b8a20d 100644 (file)
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -134,16 +134,25 @@ def pytest_collection_modifyitems(items):
 
 class NodeResult:
     def __init__(self, report=None):
-        self.outcome = None
-        self.messages = []
+        self._outcomes = {}
+        self.messages = {}
         if report is not None:
             self.update(report)
 
     def update(self, report):
-        if self.outcome is None or report.outcome != "passed":
-            self.outcome = report.outcome
-        if report.longreprtext:
-            self.messages.append(report.longreprtext)
+        # Allow the same nodeid/when to be overriden. This only happens when
+        # the test is re-run with flaky plugin. In that case, we want the
+        # latest result to override any previous results.
+        key = (report.nodeid, report.when)
+        self._outcomes[key] = report.outcome
+        self.messages[key] = report.longreprtext
+
+    @property
+    def outcome(self):
+        for outcome in self._outcomes.values():
+            if outcome != "passed":
+                return outcome
+        return "passed"
 
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
@@ -347,7 +356,7 @@ def system_test_dir(request, system_test_name, expected_artifacts):
         messages = []
         for node, result in test_results.items():
             isctest.log.debug("%s %s", result.outcome.upper(), node)
-            messages.extend(result.messages)
+            messages.extend(result.messages.values())
         for message in messages:
             isctest.log.debug("\n" + message)
         failed = any(res.outcome == "failed" for res in test_results.values())