Use CMocka generated JUnit reports where possible

Where applicable, use the more detailed CMocka generated JUnit
reports which include subtest results and timings instead of the
one generated by Meson.

Flaky tests also require retrying, so use a wrapper and mark them
with a environment variable. This is done to avoid the need to compute
an intersection of suites in Meson which is not supported out-of-the-box
(`meson test --suite=foo,bar` runs the union of foo and bar).

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eb21ca579e374a9ccbecec155875dc0afa9c034d..dfb583ce5cbf21b46ff444404d2e0740e2916fc1 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -569,21 +569,33 @@ stages:
   <<: *default_triggering_rules
   stage: unit
   # This script needs to: 1) fail if the unit tests fail, 2) fail if the
-  # junit.xml file is broken, 3) produce the junit.xml file even if the
+  # junit.xml file is broken, 3) produce the JUnit reports even if the
   # unit tests fail.  Therefore, $RET is used to "cache" the result of
   # running "meson test" as interrupting the script immediately when
   # unit tests fail would make checking the contents of the junit.xml
   # file impossible (GitLab Runner uses "set -o pipefail").
+
+  # Additionally, both flaky and CMocka test need special handling:
+  # - flaky tests are retried a number of times (default 2) before being
+  #   considered failed
+  # - for CMocka tests, we use the CMocka's XML report to get more detailed
+  #   information (subtest results and timings). Meson also produces
+  #   a JUnit report, so we need to not pass it to GitLab to avoid duplication.
   script:
     - *fips_feature_test
-    - RET=0
-    - meson test -C build --no-rebuild --no-suite flaky || RET=1
-    - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml
-    - meson test -C build --no-rebuild --suite flaky  --logbase testlog-flaky || meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || RET=1
-    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
     - *find_python
+    - *git_clone_bind9-qa
+    # Set CMocka JUnit XML output.
+    - export CMOCKA_MESSAGE_OUTPUT="xml"
+    - export CMOCKA_XML_FILE="$(pwd)/build/meson-logs/include-cmocka-%g.junit.xml"
+    - RET=0
+    - MESON_WRAPPER="$(pwd)/bind9-qa/ci/meson_retry_if_flaky.sh 2"
+    # CMocka tests: Mark Meson-generated XMLs to exclude them later with --logbase, CMocka generates better ones.
+    - meson test -C build --suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "exclude-" || RET=1
+    # Non-CMocka test: Use Meson-generated XMLs.
+    - meson test -C build --no-suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "include-" || RET=1
     - >
-      "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml
+      "$PYTHON" bind9-qa/ci/postprocess_junit_files.py build/meson-logs/include-*.junit.xml --output "$CI_PROJECT_DIR/junit.xml"
     - (exit $RET)
     - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1
   artifacts:
@@ -592,7 +604,6 @@ stages:
     reports:
       junit:
         - junit.xml
-        - build/meson-logs/testlog-flaky.junit.xml
 
 .unit_test_tsan: &unit_test_tsan_job
   <<: *unit_test_job

diff --git a/tests/isc/meson.build b/tests/isc/meson.build
index a73d8373772c908302685a361e7442ddf70d1eee..5aa7ecafeed4e35a9fcd42eaea3a1f45f39786b6 100644 (file)
--- a/tests/isc/meson.build
+++ b/tests/isc/meson.build
@@ -90,14 +90,20 @@ foreach unit : isc_test
     )
 
     suites = ['isc', 'cmocka']
+    env = environment()
+    timeout = 300
     if unit in flaky_isc_test
         suites += 'flaky'
+        # Pass FLAKY and TIMEOUT to the test wrapper so it can retry appropriately
+        env.set('FLAKY', '1')
+        env.set('TIMEOUT', timeout.to_string())
     endif
     test(
         unit,
         test_bin,
         suite: suites,
-        timeout: 300,
+        timeout: timeout,
         workdir: meson.current_source_dir(),
+        env: env,
     )
 endforeach