Generate JUnit reports for unit & system tests

This allows Gitlab to show nice summary for individual tests/test
directories and to expose the results in Gitlab API for consumption
elsewhere.

A catch: As of Gitlab 14.7.7, the detailed results are stored
only in artifacts and thus expire. All consumers (including API) need
to be "fast enough" to get the data before they disappear.
This also forces us to always store the artifacts intead of storing them
only on failure.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5877c54a2611a34bf312a3531b7fb8ba5595e691..69ba1ab230cd0e9318fe9956edff2fb0413b7792 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -323,6 +323,7 @@ stages:
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1
     - if git rev-parse > /dev/null 2>&1; then ( ! grep "^I:.*:file.*not removed$" *.log ); fi
   after_script:
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
     - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
     - test -d bind-* && cd bind-*
     - cat bin/tests/system/test-suite.log
@@ -333,7 +334,9 @@ stages:
   artifacts:
     untracked: true
     expire_in: "1 day"
-    when: on_failure
+    when: always
+    reports:
+      junit: junit.xml
 
 .system_test_gcov: &system_test_gcov_job
   <<: *system_test_common
@@ -347,10 +350,13 @@ stages:
   after_script:
     - cat bin/tests/system/test-suite.log
     - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
   artifacts:
     expire_in: "1 day"
     untracked: true
-    when: on_failure
+    when: always
+    reports:
+      junit: junit.xml
 
 .unit_test_common: &unit_test_common
   <<: *default_triggering_rules
@@ -360,6 +366,7 @@ stages:
   script:
     - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1
   after_script:
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
     - *save_out_of_tree_workspace
 
 .unit_test: &unit_test_job
@@ -367,7 +374,9 @@ stages:
   artifacts:
     untracked: true
     expire_in: "1 day"
-    when: on_failure
+    when: always
+    reports:
+      junit: junit.xml
 
 .unit_test_gcov: &unit_test_gcov_job
   <<: *unit_test_common
@@ -380,12 +389,16 @@ stages:
   <<: *unit_test_common
   after_script:
     - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
   artifacts:
     expire_in: "1 day"
     paths:
       - lib/*/tests/tsan.*
       - tsan/
-    when: on_failure
+      - junit.xml
+    when: always
+    reports:
+      junit: junit.xml
 
 .docs: &docs_job
   stage: docs

diff --git a/bin/tests/convert-trs-to-junit.py b/bin/tests/convert-trs-to-junit.py
new file mode 100755 (executable)
index 0000000..c94c037
--- /dev/null
+++ b/bin/tests/convert-trs-to-junit.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# Convert automake .trs files into JUnit format suitable for Gitlab
+
+import argparse
+import os
+import sys
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+from xml.etree.ElementTree import SubElement
+
+
+# getting explicit encoding specification right for Python 2/3 would be messy,
+# so let's hope for the best
+def read_whole_text(filename):
+    with open(filename) as inf:  # pylint: disable-msg=unspecified-encoding
+        return inf.read().strip()
+
+
+def read_trs_result(filename):
+    result = None
+    with open(filename, "r") as trs:  # pylint: disable-msg=unspecified-encoding
+        for line in trs:
+            items = line.split()
+            if len(items) < 2:
+                raise ValueError("unsupported line in trs file", filename, line)
+            if items[0] != (":test-result:"):
+                continue
+            if result is not None:
+                raise NotImplementedError("double :test-result:", filename)
+            result = items[1].upper()
+
+    if result is None:
+        raise ValueError(":test-result: not found", filename)
+
+    return result
+
+
+def find_test_relative_path(source_dir, in_path):
+    """Return {in_path}.c if it exists, with fallback to {in_path}"""
+    candidates_relative = [in_path + ".c", in_path]
+    for relative in candidates_relative:
+        absolute = os.path.join(source_dir, relative)
+        if os.path.exists(absolute):
+            return relative
+    raise KeyError
+
+
+def err_out(exception):
+    raise exception
+
+
+def walk_trss(source_dir):
+    for cur_dir, _dirs, files in os.walk(source_dir, onerror=err_out):
+        for filename in files:
+            if not filename.endswith(".trs"):
+                continue
+
+            filename_prefix = filename[: -len(".trs")]
+            log_name = filename_prefix + ".log"
+            full_trs_path = os.path.join(cur_dir, filename)
+            full_log_path = os.path.join(cur_dir, log_name)
+            sub_dir = os.path.relpath(cur_dir, source_dir)
+            test_name = os.path.join(sub_dir, filename_prefix)
+
+            t = {
+                "name": test_name,
+                "full_log_path": full_log_path,
+                "rel_log_path": os.path.relpath(full_log_path, source_dir),
+            }
+            t["result"] = read_trs_result(full_trs_path)
+
+            # try to find dir/file path for a clickable link
+            try:
+                t["rel_file_path"] = find_test_relative_path(
+                    source_dir, test_name
+                )
+            except KeyError:
+                pass  # no existing path found
+
+            yield t
+
+
+def append_testcase(testsuite, t):
+    # attributes taken from
+    # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/parsers/test/junit.rb
+    attrs = {"name": t["name"]}
+    if "rel_file_path" in t:
+        attrs["file"] = t["rel_file_path"]
+
+    testcase = SubElement(testsuite, "testcase", attrs)
+
+    # Gitlab accepts only [[ATTACHMENT| links for system-out, not raw text
+    s = SubElement(testcase, "system-out")
+    s.text = "[[ATTACHMENT|" + t["rel_log_path"] + "]]"
+    if t["result"].lower() == "pass":
+        return
+
+    # Gitlab shows output only for failed or skipped tests
+    if t["result"].lower() == "skip":
+        err = SubElement(testcase, "skipped")
+    else:
+        err = SubElement(testcase, "failure")
+    err.text = read_whole_text(t["full_log_path"])
+
+
+def gen_junit(results):
+    testsuites = Element("testsuites")
+    testsuite = SubElement(testsuites, "testsuite")
+    for test in results:
+        append_testcase(testsuite, test)
+    return testsuites
+
+
+def check_directory(path):
+    try:
+        os.listdir(path)
+        return path
+    except OSError as ex:
+        msg = "Path {} cannot be listed as a directory: {}".format(path, ex)
+        raise argparse.ArgumentTypeError(msg)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Recursively search for .trs + .log files and compile "
+        "them into JUnit XML suitable for Gitlab. Paths in the "
+        "XML are relative to the specified top directory."
+    )
+    parser.add_argument(
+        "top_directory",
+        type=check_directory,
+        help="root directory where to start scanning for .trs files",
+    )
+    args = parser.parse_args()
+    junit = gen_junit(walk_trss(args.top_directory))
+
+    # encode results into file format, on Python 3 it produces bytes
+    xml = ElementTree.tostring(junit, "utf-8")
+    # use stdout as a binary file object, Python2/3 compatibility
+    output = getattr(sys.stdout, "buffer", sys.stdout)
+    output.write(xml)
+
+
+if __name__ == "__main__":
+    main()