ci: respdiff - run diffrepro 3 times to reproduce results

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b831468ae9759ed319aa87a501d325e54e8aa114..813de76c293f236bb7f2d914e5c56f3ec7ba8532 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -206,8 +206,8 @@ respdiff:iter:udp:linux:amd64:
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh udp
     - cat results/respdiff.txt
-    - echo 'test if mismatch rate >= 1 %'
-    - grep -q '^target disagrees.*0\.[0-9][0-9] %' results/respdiff.txt
+    - echo 'test if mismatch rate < 0.8 %'
+    - grep -q '^target disagrees.*0\.[0-7][0-9] %' results/respdiff.txt
     - killall --wait kresd
     - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-udp
   dependencies:
@@ -233,8 +233,8 @@ respdiff:iter:tcp:linux:amd64:
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh tcp
     - cat results/respdiff.txt
-    - echo 'test if mismatch rate >= 1 %'
-    - grep -q '^target disagrees.*0\.[0-9][0-9] %' results/respdiff.txt
+    - echo 'test if mismatch rate < 0.8 %'
+    - grep -q '^target disagrees.*0\.[0-7][0-9] %' results/respdiff.txt
     - killall --wait kresd
     - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tcp
   dependencies:
@@ -260,8 +260,8 @@ respdiff:iter:tls:linux:amd64:
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh tls
     - cat results/respdiff.txt
-    - echo 'test if mismatch rate >= 1 %'
-    - grep -q '^target disagrees.*0\.[0-9][0-9] %' results/respdiff.txt
+    - echo 'test if mismatch rate < 0.8 %'
+    - grep -q '^target disagrees.*0\.[0-7][0-9] %' results/respdiff.txt
     - killall --wait kresd
     - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tls
   dependencies:

diff --git a/ci/respdiff/respdiff-tcp.conf b/ci/respdiff/respdiff-tcp.conf
index 87ec297f2560bf64928e5b2654425520846ec3aa..52de4fe6ccf1bb9cfaf8e8a89f2d1d6f3cce43d3 100644 (file)
--- a/ci/respdiff/respdiff-tcp.conf
+++ b/ci/respdiff/respdiff-tcp.conf
@@ -19,18 +19,21 @@ ip = 127.0.0.1
 port = 5353
 transport = tcp
 graph_color = #00a2e2
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-kresd.sh
 
 [bind]
 ip = 127.0.0.1
 port = 53533
 transport = udp
 graph_color = #e2a000
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-bind.sh
 
 [unbound]
 ip = 127.0.0.1
 port = 53535
 transport = udp
 graph_color = #218669
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-unbound.sh
 
 [diff]
 # symbolic name of server under test
@@ -45,4 +48,3 @@ criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsi
 # diffsum reports mismatches in field values in this order
 # if particular message has multiple mismatches, it is counted only once into category with highest weight
 field_weights = timeout, opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
-

diff --git a/ci/respdiff/respdiff-tls.conf b/ci/respdiff/respdiff-tls.conf
index fb3b6715a6a2d324ab72f835f0f2f1a647da892c..501f8554a291eb21d912667fda0326fec0f1bf88 100644 (file)
--- a/ci/respdiff/respdiff-tls.conf
+++ b/ci/respdiff/respdiff-tls.conf
@@ -19,18 +19,21 @@ ip = 127.0.0.1
 port = 8853
 transport = tls
 graph_color = #00a2e2
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-kresd.sh
 
 [bind]
 ip = 127.0.0.1
 port = 53533
 transport = udp
 graph_color = #e2a000
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-bind.sh
 
 [unbound]
 ip = 127.0.0.1
 port = 53535
 transport = udp
 graph_color = #218669
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-unbound.sh
 
 [diff]
 # symbolic name of server under test
@@ -45,4 +48,3 @@ criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsi
 # diffsum reports mismatches in field values in this order
 # if particular message has multiple mismatches, it is counted only once into category with highest weight
 field_weights = timeout, opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
-

diff --git a/ci/respdiff/respdiff-udp.conf b/ci/respdiff/respdiff-udp.conf
index 16d68b595c620e06d02b11e3c3211a6309deb726..cf3056de00b32f797fbbadc3c9598576d5a270c6 100644 (file)
--- a/ci/respdiff/respdiff-udp.conf
+++ b/ci/respdiff/respdiff-udp.conf
@@ -19,18 +19,21 @@ ip = 127.0.0.1
 port = 5353
 transport = udp
 graph_color = #00a2e2
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-kresd.sh
 
 [bind]
 ip = 127.0.0.1
 port = 53533
 transport = udp
 graph_color = #e2a000
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-bind.sh
 
 [unbound]
 ip = 127.0.0.1
 port = 53535
 transport = udp
 graph_color = #218669
+restart_script = /builds/knot/knot-resolver/ci/respdiff/restart-unbound.sh
 
 [diff]
 # symbolic name of server under test
@@ -45,4 +48,3 @@ criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsi
 # diffsum reports mismatches in field values in this order
 # if particular message has multiple mismatches, it is counted only once into category with highest weight
 field_weights = timeout, opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
-

diff --git a/ci/respdiff/restart-bind.sh b/ci/respdiff/restart-bind.sh
new file mode 100755 (executable)
index 0000000..89fd832
--- /dev/null
+++ b/ci/respdiff/restart-bind.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+service bind9 restart

diff --git a/ci/respdiff/restart-kresd.sh b/ci/respdiff/restart-kresd.sh
new file mode 100755 (executable)
index 0000000..dfe82cf
--- /dev/null
+++ b/ci/respdiff/restart-kresd.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+exec > /dev/null
+exec 2>&1
+
+PREFIX=$(pwd)/.local
+killall -w kresd
+rm -f '*.mdb'
+LD_LIBRARY_PATH=$PREFIX/lib $PREFIX/sbin/kresd -f 1 -q -c $(pwd)/ci/respdiff/kresd.config &>>kresd.log &
+
+# wait until socket is receiving connections
+sleep 1

diff --git a/ci/respdiff/restart-unbound.sh b/ci/respdiff/restart-unbound.sh
new file mode 100755 (executable)
index 0000000..c9525c0
--- /dev/null
+++ b/ci/respdiff/restart-unbound.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+service unbound restart

diff --git a/ci/respdiff/run-respdiff-tests.sh b/ci/respdiff/run-respdiff-tests.sh
index d314b6d0b464e612007e4b726379b8cdf12443a1..e596e5fc308b9eeb043d3fbba2dbcaed3bceb860 100755 (executable)
--- a/ci/respdiff/run-respdiff-tests.sh
+++ b/ci/respdiff/run-respdiff-tests.sh
@@ -3,6 +3,8 @@
 # respdiff scripts must be present in /var/opt/respdiff
 set -o errexit -o nounset -o xtrace
 
+NDIFFREPRO=3
+
 wget https://gitlab.labs.nic.cz/knot/knot-resolver/snippets/69/raw?inline=false -O /tmp/queries.txt
 mkdir results
 rm -rf respdiff.db
@@ -11,6 +13,9 @@ CONFIG="$(pwd)/ci/respdiff/respdiff-${1}.conf"
 /var/opt/respdiff/qprep.py respdiff.db < /tmp/queries.txt
 time /var/opt/respdiff/orchestrator.py respdiff.db -c "${CONFIG}"
 time /var/opt/respdiff/msgdiff.py respdiff.db -c "${CONFIG}"
+for i in $(seq $NDIFFREPRO); do
+       time /var/opt/respdiff/diffrepro.py -c "${CONFIG}" respdiff.db
+done
 /var/opt/respdiff/diffsum.py respdiff.db -c "${CONFIG}" > results/respdiff.txt
 /var/opt/respdiff/histogram.py respdiff.db -c "${CONFIG}" -o results/histogram.svg
 : minimize LMDB and log size so they can be effectively archived