Disable deadlines for hypothesis tests when running in CI

The times it takes to run tests CI vary significantly enough
that it makes hypothesis test reach their deadlines and fail randomly
marking the tests as flaky.

This commit disables the deadlines when running in CI.

(cherry picked from commit f55cacbbfd5eeaeb8e521895a0ffab8addeaad6f)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0779e094648ac124c22f6ddbe48b009f67409492..83769721550223f6b73ce0358ae8d616afe11212 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,6 +55,8 @@ variables:
   BIND_STRESS_TEST_OS: linux
   BIND_STRESS_TEST_ARCH: amd64
 
+  HYPOTHESIS_PROFILE: "ci"
+
 default:
   # Allow all running CI jobs to be automatically canceled when a new
   # version of a branch is pushed.

diff --git a/bin/tests/system/wildcard/tests_wildcard.py b/bin/tests/system/wildcard/tests_wildcard.py
index cd5b278d284a639390de94ee5c46ecf2dd071ecb..f304f966b31a8cc96356c5a5a5f7d206083cd047 100755 (executable)
--- a/bin/tests/system/wildcard/tests_wildcard.py
+++ b/bin/tests/system/wildcard/tests_wildcard.py
@@ -28,6 +28,7 @@ Limitations - untested properties:
     - special behavior of rdtypes like CNAME
 """
 
+import os
 import pytest
 
 pytest.importorskip("dns", minversion="2.0.0")
@@ -47,7 +48,7 @@ try:
     pytest.importorskip("hypothesis")
 except ValueError:
     pytest.importorskip("hypothesis", minversion="4.41.2")
-from hypothesis import assume, example, given
+from hypothesis import assume, example, given, settings
 
 from strategies import dns_names, dns_rdatatypes_without_meta
 import isctest.check
@@ -62,6 +63,10 @@ WILDCARD_RDATA = "192.0.2.1"
 IP_ADDR = "10.53.0.1"
 TIMEOUT = 5  # seconds, just a sanity check
 
+# Timing of hypothesis tests is flaky in the CI, so we disable deadlines.
+settings.register_profile("ci", deadline=None)
+settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default"))
+
 
 @given(name=dns_names(suffix=SUFFIX), rdtype=dns_rdatatypes_without_meta)
 def test_wildcard_rdtype_mismatch(