Disable deadlines for hypothesis tests when running in CI

The times it takes to run tests CI vary significantly enough
that it makes hypothesis test reach their deadlines and fail randomly
marking the tests as flaky.

This commit disables the deadlines when running in CI.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 72b64b843371ac9cf5b7bd127114d64cdaf4b5aa..0396e175db7438d4f727b51ec12fe9169a2ef60e 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,6 +55,8 @@ variables:
   BIND_STRESS_TEST_OS: linux
   BIND_STRESS_TEST_ARCH: amd64
 
+  HYPOTHESIS_PROFILE: "ci"
+
 default:
   # Allow all running CI jobs to be automatically canceled when a new
   # version of a branch is pushed.

diff --git a/bin/tests/system/wildcard/tests_wildcard.py b/bin/tests/system/wildcard/tests_wildcard.py
index cd5b278d284a639390de94ee5c46ecf2dd071ecb..f304f966b31a8cc96356c5a5a5f7d206083cd047 100755 (executable)
--- a/bin/tests/system/wildcard/tests_wildcard.py
+++ b/bin/tests/system/wildcard/tests_wildcard.py
@@ -28,6 +28,7 @@ Limitations - untested properties:
     - special behavior of rdtypes like CNAME
 """
 
+import os
 import pytest
 
 pytest.importorskip("dns", minversion="2.0.0")
@@ -47,7 +48,7 @@ try:
     pytest.importorskip("hypothesis")
 except ValueError:
     pytest.importorskip("hypothesis", minversion="4.41.2")
-from hypothesis import assume, example, given
+from hypothesis import assume, example, given, settings
 
 from strategies import dns_names, dns_rdatatypes_without_meta
 import isctest.check
@@ -62,6 +63,10 @@ WILDCARD_RDATA = "192.0.2.1"
 IP_ADDR = "10.53.0.1"
 TIMEOUT = 5  # seconds, just a sanity check
 
+# Timing of hypothesis tests is flaky in the CI, so we disable deadlines.
+settings.register_profile("ci", deadline=None)
+settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default"))
+
 
 @given(name=dns_names(suffix=SUFFIX), rdtype=dns_rdatatypes_without_meta)
 def test_wildcard_rdtype_mismatch(