oe-selftest: add resulttool junit test

Verify imagetests, and ptest with inlined as well as ptests with
attached log files work as expected.

Signed-off-by: Adrian Freihofer <adrian.freihofer@siemens.com>
Signed-off-by: Miroslav Cernak <miroslav.cernak@siemens.com>
Signed-off-by: Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py
index c3303f3fbbee270a819da46b20524c2acd89ee5e..aa90116795c8a6f11fd99ac3194a1bf764220ace 100644 (file)
--- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -14,6 +14,12 @@ from resulttool import regression as regression
 from resulttool import resultutils as resultutils
 from oeqa.selftest.case import OESelftestTestCase
 
+from resulttool.junit import junit_tree, PtestSummary
+import xml.etree.ElementTree as ET
+import logging
+import json
+
+
 class ResultToolTests(OESelftestTestCase):
     base_results_data = {'base_result1': {'configuration': {"TEST_TYPE": "runtime",
                                                             "TESTSERIES": "series1",
@@ -373,3 +379,176 @@ class ResultToolTests(OESelftestTestCase):
             self.logger, "A", "B", base_configuration["a"]["conf_X"], target_configuration["a"]["conf_Y"])
         self.assertDictEqual(
             result, {}, msg=f"ptests should be compared: {resultstring}")
+
+    @property
+    def _get_junit_testresults_1(self):
+        base_testresults = {
+            "a": {
+                "runtime_a-image": {
+                    "configuration": {"TEST_TYPE": "runtime", "MACHINE": "qemux86"},
+                    "result": {
+                        # Image test skipped
+                        "ptest.PtestRunnerTest.test_ptestrunner_expectfail": {
+                            "duration": 0,
+                            "log": "Cannot run ptests with @expectedFailure as ptests are required to pass",
+                            "status": "SKIPPED",
+                        },
+                        # Image test passed
+                        "ptest.PtestRunnerTest.test_ptestrunner_expectsuccess": {
+                            "duration": 7,
+                            "status": "PASSED",
+                        },
+                        # Passed and skipped tests: passed
+                        "ptestresult.package-passed.test_passed": {"status": "PASSED"},
+                        "ptestresult.package-passed.test_skipped": {
+                            "status": "SKIPPED"
+                        },
+                        # All tests are skipped: skipped
+                        "ptestresult.package-skipped.test_skipped": {
+                            "status": "SKIPPED"
+                        },
+                        # One or more errors: error
+                        "ptestresult.package-error.test_error": {"status": "ERROR"},
+                        "ptestresult.package-error.test_failed": {"status": "FAILED"},
+                        "ptestresult.package-error.test_skipped": {"status": "SKIPPED"},
+                        "ptestresult.package-error.test_passed": {"status": "PASSED"},
+                        # No error and one or more failed: failed
+                        "ptestresult.package-failed.test_failed": {"status": "FAILED"},
+                        "ptestresult.package-failed.test_passed": {"status": "PASSED"},
+                        "ptestresult.sections": {
+                            "package-passed": {
+                                "duration": "2",
+                                "log": "PASS: package-passed.test_passed\nPASS: package-passed.test_skipped\n",
+                            },
+                            "package-skipped": {
+                                "duration": "1",
+                                "log": "SKIPPED: package-skipped.test_skipped\n",
+                            },
+                            "package-error": {
+                                "duration": "4",
+                                "log": "ERROR: ERROR: package-error.test_error\nFAILED: package-error.test_failed\nSKIPPED: package-error.test_skipped\nPASSED: package-error.test_passed\n",
+                            },
+                            "package-failed": {
+                                "duration": "2",
+                                "log": "FAILED: package-failed.test_failed\nPASS: package-failed.test_passed\n",
+                            },
+                        },
+                    },
+                }
+            }
+        }
+        return base_testresults
+
+    def _dump_junit_tree(self, testresults, tree, files_name="junit"):
+        if self.logger.level <= logging.DEBUG:
+            junit_json_path = files_name + ".json"
+            with open(junit_json_path, "w") as f:
+                json.dump(testresults, f, indent=4)
+            self.logger.debug(
+                "Saved testresults json %s" % os.path.abspath(junit_json_path)
+            )
+            junit_xml_path = files_name + ".xml"
+            tree.write(junit_xml_path, encoding="UTF-8", xml_declaration=True)
+            self.logger.debug(
+                "Saved JUnit XML report as %s" % os.path.abspath(junit_xml_path)
+            )
+
+    def _check_junit_testresults_1(self, testsuites_node):
+        self.assertEqual(testsuites_node.attrib["errors"], "1")
+        self.assertEqual(testsuites_node.attrib["failures"], "1")
+        self.assertEqual(testsuites_node.attrib["skipped"], "2")
+        self.assertEqual(testsuites_node.attrib["tests"], "6")
+        self.assertEqual(testsuites_node.attrib["time"], "16")
+
+        testsuites = testsuites_node.findall("testsuite")
+        self.assertEqual(testsuites[0].attrib["name"], "runtime_a-image")
+        inner_testsuites = testsuites[0].findall("testsuite")
+        self.assertEqual(inner_testsuites[0].attrib["name"], "Image Tests")
+        self.assertEqual(inner_testsuites[1].attrib["name"], "Package Tests")
+
+        ptests_suite = testsuites_node.find(".//testsuite[@name='Package Tests']")
+        testcases = ptests_suite.findall("testcase")
+        self.assertEqual(testcases[0].attrib["name"], "package-passed")
+        self.assertEqual(testcases[1].attrib["name"], "package-skipped")
+        self.assertEqual(testcases[2].attrib["name"], "package-error")
+        self.assertEqual(testcases[3].attrib["name"], "package-failed")
+        self.assertEqual(testcases[0].attrib["time"], "2")
+        self.assertEqual(testcases[1].attrib["time"], "1")
+        self.assertEqual(testcases[2].attrib["time"], "4")
+        self.assertEqual(testcases[3].attrib["time"], "2")
+
+    def test_junit_log_inline(self):
+        testresults = self._get_junit_testresults_1
+        tree, test_logfiles = junit_tree(testresults)
+        self._dump_junit_tree(testresults, tree)
+        testsuites_node = tree.getroot()
+
+        # Verify the common part
+        self._check_junit_testresults_1(testsuites_node)
+
+        # Verify the inlined log files
+        ptests_suite = testsuites_node.find(".//testsuite[@name='Package Tests']")
+        testcases = ptests_suite.findall("testcase")
+        ptestresult_sections = testresults["a"]["runtime_a-image"]["result"][
+            "ptestresult.sections"
+        ]
+        self.assertEqual(
+            ptestresult_sections["package-passed"]["log"],
+            testcases[0].find("system-out").text,
+        )
+        self.assertEqual(
+            ptestresult_sections["package-skipped"]["log"],
+            testcases[1].find("system-out").text,
+        )
+        self.assertEqual(
+            ptestresult_sections["package-error"]["log"],
+            testcases[2].find("system-out").text,
+        )
+        self.assertEqual(
+            ptestresult_sections["package-failed"]["log"],
+            testcases[3].find("system-out").text,
+        )
+
+        # Check the ptest log messages are inline
+        self.assertDictEqual(test_logfiles, {})
+
+    def test_junit_log_attached(self):
+        testresults_1 = self._get_junit_testresults_1
+        test_logdir = "test-logs"
+        tree, test_logfiles = junit_tree(testresults_1, test_logdir)
+        self._dump_junit_tree(testresults_1, tree, "junit_attached")
+        testsuites_node = tree.getroot()
+
+        # Verify the common part
+        self._check_junit_testresults_1(testsuites_node)
+
+        # Verify the attached log files
+        ptests_suite = testsuites_node.find(".//testsuite[@name='Package Tests']")
+        testcases = ptests_suite.findall("testcase")
+        self.assertIn(
+            "[[ATTACHMENT|test-logs/package-passed.log]]",
+            testcases[0].find("system-out").text,
+        )
+        self.assertIn(
+            "[[ATTACHMENT|test-logs/package-skipped.log]]",
+            testcases[1].find("system-out").text,
+        )
+        self.assertIn(
+            "[[ATTACHMENT|test-logs/package-error.log]]",
+            testcases[2].find("system-out").text,
+        )
+        self.assertIn(
+            "[[ATTACHMENT|test-logs/package-failed.log]]",
+            testcases[3].find("system-out").text,
+        )
+
+        self.maxDiff = None
+        self.assertDictEqual(
+            test_logfiles,
+            {
+                "test-logs/package-passed.log": "PASS: package-passed.test_passed\nPASS: package-passed.test_skipped\n",
+                "test-logs/package-skipped.log": "SKIPPED: package-skipped.test_skipped\n",
+                "test-logs/package-error.log": "ERROR: ERROR: package-error.test_error\nFAILED: package-error.test_failed\nSKIPPED: package-error.test_skipped\nPASSED: package-error.test_passed\n",
+                "test-logs/package-failed.log": "FAILED: package-failed.test_failed\nPASS: package-failed.test_passed\n",
+            },
+        )