From: Remi Gacogne Date: Thu, 30 Aug 2018 17:16:27 +0000 (+0200) Subject: ixfrdist: add tests, run them in travis X-Git-Tag: dnsdist-1.3.3~93^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F6973%2Fhead;p=thirdparty%2Fpdns.git ixfrdist: add tests, run them in travis --- diff --git a/.travis.yml b/.travis.yml index b82211802d..29dc73f07f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ env: - PDNS_BUILD_PRODUCT=auth - PDNS_BUILD_PRODUCT=recursor - PDNS_BUILD_PRODUCT=dnsdist + - PDNS_BUILD_PRODUCT=ixfrdist before_script: - git describe --always --dirty=+ diff --git a/build-scripts/travis.sh b/build-scripts/travis.sh index 21d87d3195..a7b6a4b82b 100755 --- a/build-scripts/travis.sh +++ b/build-scripts/travis.sh @@ -329,6 +329,11 @@ install_auth() { run "sudo chmod 755 /etc/authbind/byport/53" } +install_ixfrdist() { + run "sudo apt-get -qq --no-install-recommends install \ + libyaml-cpp-dev" +} + install_recursor() { # recursor test requirements / setup # lua-posix is required for the ghost tests @@ -390,7 +395,6 @@ build_auth() { --enable-experimental-pkcs11 \ --enable-remotebackend-zeromq \ --enable-tools \ - --enable-ixfrdist \ --enable-unit-tests \ --enable-backend-unit-tests \ --disable-dependency-tracking \ @@ -401,6 +405,21 @@ build_auth() { run "find /tmp/pdns-install-dir -ls" } +build_ixfrdist() { + run "autoreconf -vi" + run "./configure \ + ${sanitizerflags} \ + --with-dynmodules='bind' \ + --with-modules='' \ + --enable-ixfrdist \ + --enable-unit-tests \ + --disable-dependency-tracking \ + --disable-silent-rules" + run "cd pdns" + run "make -k -j3 ixfrdist" + run "cd .." +} + build_recursor() { export PDNS_RECURSOR_DIR=$HOME/pdns_recursor # distribution build @@ -572,6 +591,12 @@ test_auth() { run "rm -f regression-tests/zones/*-slave.*" #FIXME } +test_ixfrdist(){ + run "cd regression-tests.ixfrdist" + run "IXFRDISTBIN=${TRAVIS_BUILD_DIR}/pdns/ixfrdist ./runtests -v || (cat ixfrdist.log; false)" + run "cd .." +} + test_recursor() { export PDNSRECURSOR="${PDNS_RECURSOR_DIR}/sbin/pdns_recursor" export DNSBULKTEST="/usr/bin/dnsbulktest" @@ -626,6 +651,8 @@ then sanitizerflags="${sanitizerflags} --enable-asan" elif [ "${PDNS_BUILD_PRODUCT}" = "dnsdist" ]; then sanitizerflags="${sanitizerflags} --enable-asan --enable-ubsan" + elif [ "${PDNS_BUILD_PRODUCT}" = "ixfrdist" ]; then + sanitizerflags="${sanitizerflags} --enable-asan" fi fi export CFLAGS=$compilerflags diff --git a/regression-tests.ixfrdist/ixfrdisttests.py b/regression-tests.ixfrdist/ixfrdisttests.py new file mode 100644 index 0000000000..50128d1ec9 --- /dev/null +++ b/regression-tests.ixfrdist/ixfrdisttests.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python2 + +import errno +import shutil +import os +import socket +import struct +import subprocess +import sys +import time +import unittest +import dns +import dns.message + +class IXFRDistTest(unittest.TestCase): + + _ixfrDistStartupDelay = 2.0 + _ixfrDistPort = 5342 + + _config_template = """ +listen: + - '127.0.0.1:%d' +acl: + - '127.0.0.0/8' +axfr-timeout: 20 +keep: 20 +tcp-in-threads: 10 +work-dir: 'ixfrdist.dir' +failed-soa-retry: 3 +""" + _config_domains = None + _config_params = ['_ixfrDistPort'] + + @classmethod + def startIXFRDist(cls): + print("Launching ixfrdist..") + conffile = 'ixfrdist.yml' + params = tuple([getattr(cls, param) for param in cls._config_params]) + print(params) + with open(conffile, 'w') as conf: + conf.write("# Autogenerated by ixfrdisttests.py\n") + conf.write(cls._config_template % params) + + if cls._config_domains is not None: + conf.write("domains:\n") + + for domain, master in cls._config_domains.items(): + conf.write(" - domain: %s\n" % (domain)) + conf.write(" master: %s\n" % (master)) + + ixfrdistcmd = [os.environ['IXFRDISTBIN'], '--config', conffile, '--debug'] + + logFile = 'ixfrdist.log' + with open(logFile, 'w') as fdLog: + cls._ixfrdist = subprocess.Popen(ixfrdistcmd, close_fds=True, + stdout=fdLog, stderr=fdLog) + + if 'IXFRDIST_FAST_TESTS' in os.environ: + delay = 0.5 + else: + delay = cls._ixfrDistStartupDelay + + time.sleep(delay) + + if cls._ixfrdist.poll() is not None: + cls._ixfrdist.kill() + sys.exit(cls._ixfrdist.returncode) + + @classmethod + def setUpSockets(cls): + print("Setting up UDP socket..") + cls._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + cls._sock.settimeout(2.0) + cls._sock.connect(("127.0.0.1", cls._ixfrDistPort)) + + @classmethod + def setUpClass(cls): + cls.startIXFRDist() + cls.setUpSockets() + + print("Launching tests..") + + @classmethod + def tearDownClass(cls): + cls.tearDownIXFRDist() + + @classmethod + def tearDownIXFRDist(cls): + if 'IXFRDIST_FAST_TESTS' in os.environ: + delay = 0.1 + else: + delay = 1.0 + + try: + if cls._ixfrdist: + cls._ixfrdist.terminate() + if cls._ixfrdist.poll() is None: + time.sleep(delay) + if cls._ixfrdist.poll() is None: + cls._ixfrdist.kill() + cls._ixfrdist.wait() + except OSError as e: + # There is a race-condition with the poll() and + # kill() statements, when the process is dead on the + # kill(), this is fine + if e.errno != errno.ESRCH: + raise + + @classmethod + def sendUDPQuery(cls, query, timeout=2.0, decode=True, fwparams=dict()): + if timeout: + cls._sock.settimeout(timeout) + + try: + cls._sock.send(query.to_wire()) + data = cls._sock.recv(4096) + except socket.timeout: + data = None + finally: + if timeout: + cls._sock.settimeout(None) + + message = None + if data: + if not decode: + return data + message = dns.message.from_wire(data, **fwparams) + return message + + # FIXME: sendTCPQuery and sendTCPQueryMultiResponse, when they are done reading + # should wait for a short while on the socket to see if more data is coming + # and error if it does! + @classmethod + def sendTCPQuery(cls, query, timeout=2.0): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if timeout: + sock.settimeout(timeout) + + sock.connect(("127.0.0.1", cls._ixfrDistPort)) + + try: + wire = query.to_wire() + sock.send(struct.pack("!H", len(wire))) + sock.send(wire) + data = sock.recv(2) + if data: + (datalen,) = struct.unpack("!H", data) + data = sock.recv(datalen) + except socket.timeout as e: + print("Timeout: %s" % (str(e))) + data = None + except socket.error as e: + print("Network error: %s" % (str(e))) + data = None + finally: + sock.close() + + message = None + if data: + message = dns.message.from_wire(data) + return message + + @classmethod + def sendTCPQueryMultiResponse(cls, query, timeout=2.0, count=1): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if timeout: + sock.settimeout(timeout) + + sock.connect(("127.0.0.1", cls._ixfrDistPort)) + + try: + wire = query.to_wire() + sock.send(struct.pack("!H", len(wire))) + sock.send(wire) + except socket.timeout as e: + raise Exception("Timeout: %s" % (str(e))) + except socket.error as e: + raise Exception("Network error: %s" % (str(e))) + + messages = [] + for i in range(count): + try: + data = sock.recv(2) + if data: + (datalen,) = struct.unpack("!H", data) + data = sock.recv(datalen) + messages.append(dns.message.from_wire(data)) + else: + break + except socket.timeout as e: + raise Exception("Timeout: %s" % (str(e))) + except socket.error as e: + raise Exception("Network error: %s" % (str(e))) + + return messages + + def setUp(self): + # This function is called before every tests + return + diff --git a/regression-tests.ixfrdist/requirements.txt b/regression-tests.ixfrdist/requirements.txt new file mode 100644 index 0000000000..62ed456579 --- /dev/null +++ b/regression-tests.ixfrdist/requirements.txt @@ -0,0 +1,3 @@ +dnspython +nose +git+https://github.com/PowerDNS/xfrserver.git@0.1 \ No newline at end of file diff --git a/regression-tests.ixfrdist/runtests b/regression-tests.ixfrdist/runtests new file mode 100755 index 0000000000..5a560a05d0 --- /dev/null +++ b/regression-tests.ixfrdist/runtests @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -e + +if [ ! -d .venv ]; then + if [ -z "$PYTHON" ]; then + if [ ! -z "$(python3 --version | egrep '^Python 3.[6789]' 2>/dev/null)" ]; then + # found python3.6 or better + PYTHON=python3 + else + # until we have better Linux distribution detection. + PYTHON=python2 + fi + fi + + virtualenv -p ${PYTHON} .venv +fi +. .venv/bin/activate +python -V +pip install -r requirements.txt + +if [ -z "${IXFRDISTBIN}" ]; then + IXFRDISTBIN=$(ls ../pdns/ixfrdist) +fi +export IXFRDISTBIN + +set -e +if [ "${PDNS_DEBUG}" = "YES" ]; then + set -x +fi + +rm -rf ixfrdist.dir +mkdir ixfrdist.dir + +nosetests --with-xunit $@ diff --git a/regression-tests.ixfrdist/test_IXFR.py b/regression-tests.ixfrdist/test_IXFR.py new file mode 100644 index 0000000000..8f99316dda --- /dev/null +++ b/regression-tests.ixfrdist/test_IXFR.py @@ -0,0 +1,114 @@ +import dns +import time + +from ixfrdisttests import IXFRDistTest +from xfrserver.xfrserver import AXFRServer + +zones = { + 1: """ +$ORIGIN example. +@ 86400 SOA foo bar 1 2 3 4 5 +@ 4242 NS ns1.example. +@ 4242 NS ns2.example. +ns1.example. 4242 A 192.0.2.1 +ns2.example. 4242 A 192.0.2.2 +""", + 2: """ +$ORIGIN example. +@ 86400 SOA foo bar 2 2 3 4 5 +@ 4242 NS ns1.example. +@ 4242 NS ns2.example. +ns1.example. 4242 A 192.0.2.1 +ns2.example. 4242 A 192.0.2.2 +newrecord.example. 8484 A 192.0.2.42 +""" +} + + +xfrServerPort = 4244 +xfrServer = AXFRServer(xfrServerPort, zones) + +class IXFRDistBasicTest(IXFRDistTest): + """ + This test makes sure that we correctly fetch a zone via AXFR, and provide the full AXFR and IXFR + """ + + global xfrServerPort + _xfrDone = 0 + _config_domains = { 'example': '127.0.0.1:' + str(xfrServerPort) } + + @classmethod + def setUpClass(cls): + + cls.startIXFRDist() + cls.setUpSockets() + + @classmethod + def tearDownClass(cls): + cls.tearDownIXFRDist() + + def waitUntilCorrectSerialIsLoaded(self, serial, timeout=10): + global xfrServer + + xfrServer.moveToSerial(serial) + + attempts = 0 + while attempts < timeout: + print('attempts=%s timeout=%s' % (attempts, timeout)) + servedSerial = xfrServer.getServedSerial() + print('servedSerial=%s' % servedSerial) + if servedSerial > serial: + raise AssertionError("Expected serial %d, got %d" % (serial, servedSerial)) + if servedSerial == serial: + self._xfrDone = self._xfrDone + 1 + return + + attempts = attempts + 1 + time.sleep(1) + + raise AssertionError("Waited %d seconds for the serial to be updated to %d but the last served serial is still %d" % (timeout, serial, servedSerial)) + + def checkFullZone(self, serial): + global zones + + # FIXME: 90% duplication from _getRecordsForSerial + zone = [] + for i in dns.zone.from_text(zones[serial], relativize=False).iterate_rdatasets(): + n, rds = i + rrs=dns.rrset.RRset(n, rds.rdclass, rds.rdtype) + rrs.update(rds) + zone.append(rrs) + + expected =[[zone[0]], sorted(zone[1:], key=lambda rrset: (rrset.name, rrset.rdtype)), [zone[0]]] # AXFRs are SOA-wrapped + + query = dns.message.make_query('example.', 'AXFR') + res = self.sendTCPQueryMultiResponse(query, count=len(expected)+1) # +1 for trailing data check + answers = [r.answer for r in res] + answers[1].sort(key=lambda rrset: (rrset.name, rrset.rdtype)) + self.assertEqual(answers, expected) + + def checkIXFR(self, fromserial, toserial): + global zones, xfrServer + + ixfr = [] + soa1 = xfrServer._getSOAForSerial(fromserial) + soa2 = xfrServer._getSOAForSerial(toserial) + newrecord = [r for r in xfrServer._getRecordsForSerial(toserial) if r.name==dns.name.from_text('newrecord.example.')] + query = dns.message.make_query('example.', 'IXFR') + query.authority = [soa1] + + expected = [[soa2], [soa1], [soa2], newrecord, [soa2]] + res = self.sendTCPQueryMultiResponse(query, count=len(expected)+1) # +1 for trailing data check + answers = [r.answer for r in res] + + # answers[1].sort(key=lambda rrset: (rrset.name, rrset.rdtype)) + self.assertEqual(answers, expected) + + def testXFR(self): + self.waitUntilCorrectSerialIsLoaded(1) + self.checkFullZone(1) + + self.waitUntilCorrectSerialIsLoaded(2) + self.checkFullZone(2) + + self.checkIXFR(1,2) \ No newline at end of file