Set stricter limits for respdiff testing

Adjust the limit of maximum disagreements in respdiff results based on
recent pipeline results.

The respdiff and respdiff:asan seem to have almost identical results,
typically around 0.07 % of differences with ocassional spikes up to
around 0.11 %. Similar results are for respdiff:tsan, perhaps with more
common spikes with values up to around 0.12 %. Set the limit to 0.15 %
to allow for some tolerance due to network conditions, time of day etc.

The respdiff:third-party has a slightly higher disagreements average,
with typical values being around 0.12 %. Set the limit to 0.2 %.

Exceeding either of those values should be quite clear indication that
some resolution behaviour has changed, since the values appear to be
very stable within the newly configured limits.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 397451fd01bd051e04c952ab3ee5582508a22e25..2ccaf19244681836f5905d2956e76a7c9c4aaa2b 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1596,7 +1596,7 @@ respdiff:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -Og -DISC_TRACK_PTHREADS_OBJECTS"
-    MAX_DISAGREEMENTS_PERCENTAGE: "0.5"
+    MAX_DISAGREEMENTS_PERCENTAGE: "0.15"
   script:
     - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named"
 
@@ -1609,7 +1609,7 @@ respdiff:asan:
     CFLAGS: "${CFLAGS_COMMON} -Og -fsanitize=address,undefined"
     LDFLAGS: "-fsanitize=address,undefined"
     EXTRA_CONFIGURE: "--without-jemalloc"
-    MAX_DISAGREEMENTS_PERCENTAGE: "0.5"
+    MAX_DISAGREEMENTS_PERCENTAGE: "0.15"
   script:
     - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named"
 
@@ -1622,7 +1622,7 @@ respdiff:tsan:
     CFLAGS: "${CFLAGS_COMMON} -Og -fsanitize=thread"
     LDFLAGS: "-fsanitize=thread"
     EXTRA_CONFIGURE: "--enable-pthread-rwlock --without-jemalloc PKG_CONFIG_PATH=/opt/tsan/lib/pkgconfig"
-    MAX_DISAGREEMENTS_PERCENTAGE: "0.5"
+    MAX_DISAGREEMENTS_PERCENTAGE: "0.15"
     TSAN_OPTIONS: "${TSAN_OPTIONS_DEBIAN}"
   script:
     - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named"
@@ -1637,7 +1637,7 @@ respdiff-third-party:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -Og"
-    MAX_DISAGREEMENTS_PERCENTAGE: "0.5"
+    MAX_DISAGREEMENTS_PERCENTAGE: "0.2"
   script:
     - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}"