Allow flaky unit tests to be re-run in CI

Mark instable unit tests with `flaky` test suite. Execute the stable
separately in CI. Allow the flaky ones to be re-executed once in case
they fail.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ed35458681cd65532245a58619a45710102805cf..98818b9d0717a4b4e6b23e7f13c28df7fcbfd75b 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -454,8 +454,9 @@ stages:
   script:
     - *fips_feature_test
     - RET=0
-    - meson test -C build --no-rebuild || RET=1
+    - meson test -C build --no-rebuild --no-suite flaky || RET=1
     - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml
+    - meson test -C build --no-rebuild --suite flaky || meson test -C build --no-rebuild --suite flaky || RET=1
     - *check_for_junit_xml
     - (exit $RET)
     - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1
@@ -463,7 +464,9 @@ stages:
     untracked: true
     when: always
     reports:
-      junit: junit.xml
+      junit:
+        - junit.xml
+        - build/meson-logs/testlog-flaky.junit.xml
 
 .unit_test_tsan: &unit_test_tsan_job
   <<: *unit_test_job

diff --git a/tests/isc/meson.build b/tests/isc/meson.build
index 78da74ac1164b998c2209ae61bac3236d1ca15e4..10a14da6f86cbfbd55bb19f02b5982a562f221f9 100644 (file)
--- a/tests/isc/meson.build
+++ b/tests/isc/meson.build
@@ -57,6 +57,11 @@ isc_test = [
     'work',
 ]
 
+flaky_isc_test = [
+    'proxystream',
+    'random',
+]
+
 if config.has('HAVE_NGHTTP2')
     isc_test += 'doh'
 endif
@@ -84,10 +89,14 @@ foreach unit : isc_test
         ],
     )
 
+    suites = ['isc']
+    if unit in flaky_isc_test
+        suites += 'flaky'
+    endif
     test(
         unit,
         test_bin,
-        suite: 'isc',
+        suite: suites,
         timeout: 300,
         workdir: meson.current_source_dir(),
     )