CI: migrate to respdiff v2

All resolvers are configured as recursors (no forwarding).
We should extend testing so we can test recursion and forwarding at the
same time.

List of queries is taken dynamically from
https://gitlab.labs.nic.cz/knot/knot-resolver/snippets/69
This will likely change in future.

The test will fail if mismatch rate >= 1 %.
This is error margin for network instability and few possibly broken upstreams
in the test list.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5d0e773415e91f606144eaa5cfe177637fac17b3..8f24cf6c7b93768b40f12dafd23f6574bdf283c4 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -44,21 +44,22 @@ respdiff:linux:amd64:
   image: cznic/ubuntu-respdif:16.04
   stage: test
   script:
-    - LD_LIBRARY_PATH=$(pwd)/.local/lib /home/kresdbench/run.sh $(pwd)/.local/sbin/kresd
-    - /home/kresdbench/resolver-benchmarking/response_differences/nightly.sh
-    - cp -r /home/kresdbench/resolver-benchmarking/response_differences/results $(pwd)
+    - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
+    - ./ci/respdiff/run-respdiff-tests.sh
+    - cat ./results/respdiff.txt
+    - echo 'test if mismatch rate >= 1 %'
+    - grep -q '^target diagrees.*0\.[0-9][0-9] %' ./results/respdiff.txt
   artifacts:
     when: always
     expire_in: '1 week'
     paths:
-      - results/*.json
-      - results/*.out
-      - results/*.log
+      - results/*.txt
   tags:
     - docker
     - linux
     - amd64
-  allow_failure: true
+  dependencies:
+    - build:linux:amd64
 
 #arm_build:
 #  image: cznic/armhf-ubuntu:16.04

diff --git a/ci/respdiff/kresd.config b/ci/respdiff/kresd.config
new file mode 100644 (file)
index 0000000..a0476a6
--- /dev/null
+++ b/ci/respdiff/kresd.config
@@ -0,0 +1,25 @@
+-- Refer to manual: https://knot-resolver.readthedocs.io/en/latest/daemon.html#configuration
+
+-- Listen on localhost and external interface
+net.listen('127.0.0.1', 5353)
+net.listen('::1', 5353)
+
+-- Auto-maintain root TA
+trust_anchors.file = '.local/etc/kresd/root.keys'
+
+-- Large cache size, so we don't need to flush often
+-- This can be larger than available RAM, least frequently accessed
+-- records will be paged out
+cache.size = 1024 * MB
+
+-- Load Useful modules
+modules = {
+       'workarounds < iterate',
+       'policy',   -- Block queries to local zones/bad sites
+       'view',     -- Views for certain clients
+       'hints',    -- Load /etc/hosts and allow custom root hints
+       'stats',    -- Track internal statistics
+}
+
+
+verbose(false)

diff --git a/ci/respdiff/respdiff.conf b/ci/respdiff/respdiff.conf
new file mode 100644 (file)
index 0000000..7ce9a84
--- /dev/null
+++ b/ci/respdiff/respdiff.conf
@@ -0,0 +1,39 @@
+[sendrecv]
+# in seconds
+timeout = 5
+# number of queries to run simultaneously
+jobs = 64
+
+[servers]
+names = kresd, bind, unbound
+# symbolic names of DNS servers under test
+# separate multiple values by ,
+
+# each symbolic name in [servers] section refers to config section
+# containing IP address and port of particular server
+[kresd]
+ip = ::1
+port = 5353
+
+[bind]
+ip = 127.0.0.1
+port = 53533
+
+[unbound]
+ip = 127.0.0.1
+port = 53535
+
+[diff]
+# symbolic name of server under test
+# other servers are used as reference when comparing answers from the target
+target = kresd
+
+# fields and comparison methods used when comparing two DNS messages
+criteria = opcode, rcode, flags, question, qname, qtype, answertypes, answerrrsigs
+# other supported criteria values: authority, additional, edns, nsid
+
+[report]
+# diffsum reports mismatches in field values in this order
+# if particular message has multiple mismatches, it is counted only once into category with highest weight
+field_weights = opcode, qcase, qtype, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
+

diff --git a/ci/respdiff/run-respdiff-tests.sh b/ci/respdiff/run-respdiff-tests.sh
new file mode 100755 (executable)
index 0000000..9cde3d6
--- /dev/null
+++ b/ci/respdiff/run-respdiff-tests.sh
@@ -0,0 +1,7 @@
+wget https://gitlab.labs.nic.cz/knot/knot-resolver/snippets/69/raw?inline=false -O /tmp/queries.txt
+mkdir results;
+rm -rf /tmp/respdiff;
+python3 /var/opt/respdiff/qprep.py /tmp/respdiff < /tmp/queries.txt && \
+python3 /var/opt/respdiff/orchestrator.py /tmp/respdiff -c $(pwd)/ci/respdiff/respdiff.conf && \
+python3 /var/opt/respdiff/msgdiff.py /tmp/respdiff -c $(pwd)/ci/respdiff/respdiff.conf && \
+python3 /var/opt/respdiff/diffsum.py /tmp/respdiff -c $(pwd)/ci/respdiff/respdiff.conf > results/respdiff.txt

diff --git a/ci/respdiff/start-resolvers.sh b/ci/respdiff/start-resolvers.sh
new file mode 100755 (executable)
index 0000000..f8b7c44
--- /dev/null
+++ b/ci/respdiff/start-resolvers.sh
@@ -0,0 +1,11 @@
+#run unbound
+service unbound start && service unbound status;
+# dig @localhost -p 53535
+
+#run bind
+service bind9 start && service bind9 status;
+# dig @localhost -p 53533
+
+#run kresd
+LD_LIBRARY_PATH=$PREFIX/lib $PREFIX/sbin/kresd -f 1 -q -c $(pwd)/ci/respdiff/kresd.config &
+# dig @localhost -p 5353