From: Adrian Freihofer Date: Thu, 8 Jan 2026 11:36:45 +0000 (+0100) Subject: oe-selftest: add resulttool junit test X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1b231aeabfb83dcd44412a8a39c857d33a399550;p=thirdparty%2Fopenembedded%2Fopenembedded-core.git oe-selftest: add resulttool junit test Verify imagetests, and ptest with inlined as well as ptests with attached log files work as expected. Signed-off-by: Adrian Freihofer Signed-off-by: Miroslav Cernak Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index c3303f3fbb..aa90116795 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -14,6 +14,12 @@ from resulttool import regression as regression from resulttool import resultutils as resultutils from oeqa.selftest.case import OESelftestTestCase +from resulttool.junit import junit_tree, PtestSummary +import xml.etree.ElementTree as ET +import logging +import json + + class ResultToolTests(OESelftestTestCase): base_results_data = {'base_result1': {'configuration': {"TEST_TYPE": "runtime", "TESTSERIES": "series1", @@ -373,3 +379,176 @@ class ResultToolTests(OESelftestTestCase): self.logger, "A", "B", base_configuration["a"]["conf_X"], target_configuration["a"]["conf_Y"]) self.assertDictEqual( result, {}, msg=f"ptests should be compared: {resultstring}") + + @property + def _get_junit_testresults_1(self): + base_testresults = { + "a": { + "runtime_a-image": { + "configuration": {"TEST_TYPE": "runtime", "MACHINE": "qemux86"}, + "result": { + # Image test skipped + "ptest.PtestRunnerTest.test_ptestrunner_expectfail": { + "duration": 0, + "log": "Cannot run ptests with @expectedFailure as ptests are required to pass", + "status": "SKIPPED", + }, + # Image test passed + "ptest.PtestRunnerTest.test_ptestrunner_expectsuccess": { + "duration": 7, + "status": "PASSED", + }, + # Passed and skipped tests: passed + "ptestresult.package-passed.test_passed": {"status": "PASSED"}, + "ptestresult.package-passed.test_skipped": { + "status": "SKIPPED" + }, + # All tests are skipped: skipped + "ptestresult.package-skipped.test_skipped": { + "status": "SKIPPED" + }, + # One or more errors: error + "ptestresult.package-error.test_error": {"status": "ERROR"}, + "ptestresult.package-error.test_failed": {"status": "FAILED"}, + "ptestresult.package-error.test_skipped": {"status": "SKIPPED"}, + "ptestresult.package-error.test_passed": {"status": "PASSED"}, + # No error and one or more failed: failed + "ptestresult.package-failed.test_failed": {"status": "FAILED"}, + "ptestresult.package-failed.test_passed": {"status": "PASSED"}, + "ptestresult.sections": { + "package-passed": { + "duration": "2", + "log": "PASS: package-passed.test_passed\nPASS: package-passed.test_skipped\n", + }, + "package-skipped": { + "duration": "1", + "log": "SKIPPED: package-skipped.test_skipped\n", + }, + "package-error": { + "duration": "4", + "log": "ERROR: ERROR: package-error.test_error\nFAILED: package-error.test_failed\nSKIPPED: package-error.test_skipped\nPASSED: package-error.test_passed\n", + }, + "package-failed": { + "duration": "2", + "log": "FAILED: package-failed.test_failed\nPASS: package-failed.test_passed\n", + }, + }, + }, + } + } + } + return base_testresults + + def _dump_junit_tree(self, testresults, tree, files_name="junit"): + if self.logger.level <= logging.DEBUG: + junit_json_path = files_name + ".json" + with open(junit_json_path, "w") as f: + json.dump(testresults, f, indent=4) + self.logger.debug( + "Saved testresults json %s" % os.path.abspath(junit_json_path) + ) + junit_xml_path = files_name + ".xml" + tree.write(junit_xml_path, encoding="UTF-8", xml_declaration=True) + self.logger.debug( + "Saved JUnit XML report as %s" % os.path.abspath(junit_xml_path) + ) + + def _check_junit_testresults_1(self, testsuites_node): + self.assertEqual(testsuites_node.attrib["errors"], "1") + self.assertEqual(testsuites_node.attrib["failures"], "1") + self.assertEqual(testsuites_node.attrib["skipped"], "2") + self.assertEqual(testsuites_node.attrib["tests"], "6") + self.assertEqual(testsuites_node.attrib["time"], "16") + + testsuites = testsuites_node.findall("testsuite") + self.assertEqual(testsuites[0].attrib["name"], "runtime_a-image") + inner_testsuites = testsuites[0].findall("testsuite") + self.assertEqual(inner_testsuites[0].attrib["name"], "Image Tests") + self.assertEqual(inner_testsuites[1].attrib["name"], "Package Tests") + + ptests_suite = testsuites_node.find(".//testsuite[@name='Package Tests']") + testcases = ptests_suite.findall("testcase") + self.assertEqual(testcases[0].attrib["name"], "package-passed") + self.assertEqual(testcases[1].attrib["name"], "package-skipped") + self.assertEqual(testcases[2].attrib["name"], "package-error") + self.assertEqual(testcases[3].attrib["name"], "package-failed") + self.assertEqual(testcases[0].attrib["time"], "2") + self.assertEqual(testcases[1].attrib["time"], "1") + self.assertEqual(testcases[2].attrib["time"], "4") + self.assertEqual(testcases[3].attrib["time"], "2") + + def test_junit_log_inline(self): + testresults = self._get_junit_testresults_1 + tree, test_logfiles = junit_tree(testresults) + self._dump_junit_tree(testresults, tree) + testsuites_node = tree.getroot() + + # Verify the common part + self._check_junit_testresults_1(testsuites_node) + + # Verify the inlined log files + ptests_suite = testsuites_node.find(".//testsuite[@name='Package Tests']") + testcases = ptests_suite.findall("testcase") + ptestresult_sections = testresults["a"]["runtime_a-image"]["result"][ + "ptestresult.sections" + ] + self.assertEqual( + ptestresult_sections["package-passed"]["log"], + testcases[0].find("system-out").text, + ) + self.assertEqual( + ptestresult_sections["package-skipped"]["log"], + testcases[1].find("system-out").text, + ) + self.assertEqual( + ptestresult_sections["package-error"]["log"], + testcases[2].find("system-out").text, + ) + self.assertEqual( + ptestresult_sections["package-failed"]["log"], + testcases[3].find("system-out").text, + ) + + # Check the ptest log messages are inline + self.assertDictEqual(test_logfiles, {}) + + def test_junit_log_attached(self): + testresults_1 = self._get_junit_testresults_1 + test_logdir = "test-logs" + tree, test_logfiles = junit_tree(testresults_1, test_logdir) + self._dump_junit_tree(testresults_1, tree, "junit_attached") + testsuites_node = tree.getroot() + + # Verify the common part + self._check_junit_testresults_1(testsuites_node) + + # Verify the attached log files + ptests_suite = testsuites_node.find(".//testsuite[@name='Package Tests']") + testcases = ptests_suite.findall("testcase") + self.assertIn( + "[[ATTACHMENT|test-logs/package-passed.log]]", + testcases[0].find("system-out").text, + ) + self.assertIn( + "[[ATTACHMENT|test-logs/package-skipped.log]]", + testcases[1].find("system-out").text, + ) + self.assertIn( + "[[ATTACHMENT|test-logs/package-error.log]]", + testcases[2].find("system-out").text, + ) + self.assertIn( + "[[ATTACHMENT|test-logs/package-failed.log]]", + testcases[3].find("system-out").text, + ) + + self.maxDiff = None + self.assertDictEqual( + test_logfiles, + { + "test-logs/package-passed.log": "PASS: package-passed.test_passed\nPASS: package-passed.test_skipped\n", + "test-logs/package-skipped.log": "SKIPPED: package-skipped.test_skipped\n", + "test-logs/package-error.log": "ERROR: ERROR: package-error.test_error\nFAILED: package-error.test_failed\nSKIPPED: package-error.test_skipped\nPASSED: package-error.test_passed\n", + "test-logs/package-failed.log": "FAILED: package-failed.test_failed\nPASS: package-failed.test_passed\n", + }, + )