From: Štěpán Balážik <stepan@isc.org>
Date: Wed, 15 Oct 2025 17:23:59 +0000 (+0200)
Subject: Use CMocka generated JUnit reports where possible
X-Git-Tag: v9.21.17~18^2
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=237489caf;p=thirdparty%2Fbind9.git

Use CMocka generated JUnit reports where possible

Where applicable, use the more detailed CMocka generated JUnit
reports which include subtest results and timings instead of the
one generated by Meson.

Flaky tests also require retrying, so use a wrapper and mark them
with a environment variable. This is done to avoid the need to compute
an intersection of suites in Meson which is not supported out-of-the-box
(`meson test --suite=foo,bar` runs the union of foo and bar).
---

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eb21ca579e3..dfb583ce5cb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -569,21 +569,33 @@ stages:
   <<: *default_triggering_rules
   stage: unit
   # This script needs to: 1) fail if the unit tests fail, 2) fail if the
-  # junit.xml file is broken, 3) produce the junit.xml file even if the
+  # junit.xml file is broken, 3) produce the JUnit reports even if the
   # unit tests fail.  Therefore, $RET is used to "cache" the result of
   # running "meson test" as interrupting the script immediately when
   # unit tests fail would make checking the contents of the junit.xml
   # file impossible (GitLab Runner uses "set -o pipefail").
+
+  # Additionally, both flaky and CMocka test need special handling:
+  # - flaky tests are retried a number of times (default 2) before being
+  #   considered failed
+  # - for CMocka tests, we use the CMocka's XML report to get more detailed
+  #   information (subtest results and timings). Meson also produces
+  #   a JUnit report, so we need to not pass it to GitLab to avoid duplication.
   script:
     - *fips_feature_test
-    - RET=0
-    - meson test -C build --no-rebuild --no-suite flaky || RET=1
-    - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml
-    - meson test -C build --no-rebuild --suite flaky  --logbase testlog-flaky || meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || RET=1
-    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
     - *find_python
+    - *git_clone_bind9-qa
+    # Set CMocka JUnit XML output.
+    - export CMOCKA_MESSAGE_OUTPUT="xml"
+    - export CMOCKA_XML_FILE="$(pwd)/build/meson-logs/include-cmocka-%g.junit.xml"
+    - RET=0
+    - MESON_WRAPPER="$(pwd)/bind9-qa/ci/meson_retry_if_flaky.sh 2"
+    # CMocka tests: Mark Meson-generated XMLs to exclude them later with --logbase, CMocka generates better ones.
+    - meson test -C build --suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "exclude-" || RET=1
+    # Non-CMocka test: Use Meson-generated XMLs.
+    - meson test -C build --no-suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "include-" || RET=1
     - >
-      "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml
+      "$PYTHON" bind9-qa/ci/postprocess_junit_files.py build/meson-logs/include-*.junit.xml --output "$CI_PROJECT_DIR/junit.xml"
     - (exit $RET)
     - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1
   artifacts:
@@ -592,7 +604,6 @@ stages:
     reports:
       junit:
         - junit.xml
-        - build/meson-logs/testlog-flaky.junit.xml
 
 .unit_test_tsan: &unit_test_tsan_job
   <<: *unit_test_job
diff --git a/tests/isc/meson.build b/tests/isc/meson.build
index a73d8373772..5aa7ecafeed 100644
--- a/tests/isc/meson.build
+++ b/tests/isc/meson.build
@@ -90,14 +90,20 @@ foreach unit : isc_test
     )
 
     suites = ['isc', 'cmocka']
+    env = environment()
+    timeout = 300
     if unit in flaky_isc_test
         suites += 'flaky'
+        # Pass FLAKY and TIMEOUT to the test wrapper so it can retry appropriately
+        env.set('FLAKY', '1')
+        env.set('TIMEOUT', timeout.to_string())
     endif
     test(
         unit,
         test_bin,
         suite: suites,
-        timeout: 300,
+        timeout: timeout,
         workdir: meson.current_source_dir(),
+        env: env,
     )
 endforeach