import sys
import time
+auth_backend_ip_addr = os.getenv('AUTH_BACKEND_IP_ADDR', '127.0.0.1')
+
+clang_version = os.getenv('CLANG_VERSION', '13')
+rust_version = 'rust-1.72.0-x86_64-unknown-linux-gnu'
+
all_build_deps = [
'ccache',
'libboost-all-dev',
'libcap2',
'libfstrm0',
'libluajit-5.1-2',
- 'libsnmp35',
+ '"libsnmp[1-9]+"',
'libsodium23',
'libssl1.1',
'libsystemd0',
'curl',
'default-jre-headless',
'dnsutils',
- 'docker-compose',
'faketime',
'gawk',
'krb5-user',
'ldnsutils',
- 'libboost-serialization1.71.0',
+ '"libboost-serialization1.7[1-9]+"',
'libcdb1',
'libcurl4',
'libgeoip1',
def apt_fresh(c):
c.sudo('sed -i \'s/azure\.//\' /etc/apt/sources.list')
c.sudo('apt-get update')
- c.sudo('apt-get -qq -y --allow-downgrades dist-upgrade')
+ c.sudo('apt-get -y --allow-downgrades dist-upgrade')
@task
def install_clang(c):
"""
- install clang-12 and llvm-12
+ install clang and llvm
"""
- c.sudo('apt-get -qq -y --no-install-recommends install clang-12 llvm-12')
+ c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version} llvm-{clang_version}')
+
+@task
+def install_clang_tidy_tools(c):
+ c.sudo(f'apt-get -y --no-install-recommends install clang-tidy-{clang_version} clang-tools-{clang_version} bear python3-yaml')
@task
def install_clang_runtime(c):
# this gives us the symbolizer, for symbols in asan/ubsan traces
- c.sudo('apt-get -qq -y --no-install-recommends install clang-12')
+ c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version}')
+
+@task
+def ci_install_rust(c, repo):
+ c.sudo(f'{repo}/builder-support/helpers/install_rust.sh {rust_version}')
def install_libdecaf(c, product):
c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf')
with c.cd('/tmp/libdecaf'):
c.run('git checkout 41f349')
- c.run('CC=clang-12 CXX=clang-12 '
+ c.run(f'CC=clang-{clang_version} CXX=clang-{clang_version} '
'cmake -B build '
'-DCMAKE_INSTALL_PREFIX=/usr/local '
'-DCMAKE_INSTALL_LIBDIR=lib '
@task
def install_doc_deps(c):
- c.sudo('apt-get install -qq -y ' + ' '.join(doc_deps))
+ c.sudo('apt-get install -y ' + ' '.join(doc_deps))
@task
def install_doc_deps_pdf(c):
- c.sudo('apt-get install -qq -y ' + ' '.join(doc_deps_pdf))
+ c.sudo('apt-get install -y ' + ' '.join(doc_deps_pdf))
@task
def install_auth_build_deps(c):
- c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps))
+ c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps))
install_libdecaf(c, 'pdns-auth')
def setup_authbind(c):
extra=[]
for b in backend:
extra.extend(auth_backend_test_deps[b])
- c.sudo('apt-get -y -qq install ' + ' '.join(extra+auth_test_deps))
+ c.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install ' + ' '.join(extra+auth_test_deps))
c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*')
# c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then
@task
def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get
- c.sudo('apt-get --no-install-recommends -qq -y install ' + ' '.join(rec_bulk_deps))
+ c.sudo('apt-get --no-install-recommends -y install ' + ' '.join(rec_bulk_deps))
c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
@task
def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get
- c.sudo('apt-get --no-install-recommends install -qq -y ' + ' '.join(rec_bulk_deps) + ' \
+ c.sudo('apt-get --no-install-recommends install -y ' + ' '.join(rec_bulk_deps) + ' \
pdns-server pdns-backend-bind daemontools \
jq libfaketime lua-posix lua-socket bc authbind \
python3-venv python3-dev default-libmysqlclient-dev libpq-dev \
setup_authbind(c)
c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
- c.sudo('systemctl restart snmpd')
+ c.sudo('/etc/init.d/snmpd restart')
time.sleep(5)
c.sudo('chmod 755 /var/agentx')
@task
def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt-get
- c.sudo('apt-get install -qq -y \
+ c.sudo('apt-get install -y \
libluajit-5.1-2 \
libboost-all-dev \
libcap2 \
libh2o-evloop0.13 \
liblmdb0 \
libnghttp2-14 \
- libre2-5 \
+ "libre2-[1-9]+" \
libssl-dev \
libsystemd0 \
libsodium23 \
protobuf-compiler \
python3-venv snmpd prometheus')
c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
- c.sudo('systemctl restart snmpd')
+ c.sudo('/etc/init.d/snmpd restart')
time.sleep(5)
c.sudo('chmod 755 /var/agentx')
@task
def install_rec_build_deps(c):
- c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + rec_build_deps))
+ c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + rec_build_deps))
@task
def install_dnsdist_build_deps(c):
- c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps))
+ c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps))
@task
def ci_autoconf(c):
- c.run('BUILDER_VERSION=0.0.0-git1 autoreconf -vfi')
+ c.run('autoreconf -vfi')
@task
def ci_docs_build(c):
f'CFLAGS="{get_cflags()}"',
f'CXXFLAGS="{get_cxxflags()}"',
'./configure',
- "CC='clang-12'",
- "CXX='clang++-12'",
+ f"CC='clang-{clang_version}'",
+ f"CXX='clang++-{clang_version}'",
"--enable-option-checking=fatal",
"--enable-systemd",
"--with-libsodium",
--enable-systemd \
--prefix=/opt/dnsdist \
--with-gnutls \
+ --with-h2o \
--with-libsodium \
--with-lua=luajit \
--with-libcap \
+ --with-net-snmp \
--with-nghttp2 \
--with-re2 '
else:
--without-cdb \
--without-ebpf \
--without-gnutls \
+ --without-h2o \
--without-libedit \
--without-libsodium \
--without-lmdb \
-DDISABLE_FALSE_SHARING_PADDING \
-DDISABLE_NPN'
unittests = ' --enable-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
+ fuzztargets = '--enable-fuzz-targets' if os.getenv('FUZZING_TARGETS') == 'yes' else ''
sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
cflags = '-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int'
cxxflags = cflags + ' -Wp,-D_GLIBCXX_ASSERTIONS ' + additional_flags
- res = c.run('''CFLAGS="%s" \
+ res = c.run(f'''CFLAGS="%s" \
CXXFLAGS="%s" \
- AR=llvm-ar-12 \
- RANLIB=llvm-ranlib-12 \
+ AR=llvm-ar-{clang_version} \
+ RANLIB=llvm-ranlib-{clang_version} \
./configure \
- CC='clang-12' \
- CXX='clang++-12' \
+ CC='clang-{clang_version}' \
+ CXX='clang++-{clang_version}' \
--enable-option-checking=fatal \
--enable-fortify-source=auto \
--enable-auto-var-init=pattern \
--enable-lto=thin \
- --prefix=/opt/dnsdist %s %s %s''' % (cflags, cxxflags, features_set, sanitizers, unittests), warn=True)
+ --prefix=/opt/dnsdist %s %s %s %s''' % (cflags, cxxflags, features_set, sanitizers, unittests, fuzztargets), warn=True)
if res.exited != 0:
c.run('cat config.log')
raise UnexpectedExit(res)
def ci_auth_make(c):
c.run('make -j8 -k V=1')
+@task
+def ci_auth_make_bear(c):
+ # Needed for clang-tidy -line-filter vs project structure shenanigans
+ with c.cd('pdns'):
+ c.run('bear --append -- make -j8 -k V=1 -C ..')
+
@task
def ci_rec_make(c):
c.run('make -j8 -k V=1')
+@task
+def ci_rec_make_bear(c):
+ # Assumed to be running under ./pdns/recursordist/
+ c.run('bear --append -- make -j8 -k V=1')
+
@task
def ci_dnsdist_make(c):
c.run('make -j4 -k V=1')
+@task
+def ci_dnsdist_make_bear(c):
+ # Assumed to be running under ./pdns/dnsdistdist/
+ c.run('bear --append -- make -j4 -k V=1')
+
@task
def ci_auth_install_remotebackend_test_deps(c):
with c.cd('modules/remotebackend'):
# c.run('bundle config set path vendor/bundle')
c.run('sudo ruby -S bundle install')
- c.sudo('apt-get install -qq -y socat')
+ c.sudo('apt-get install -y socat')
@task
def ci_auth_run_unit_tests(c):
c.run('cat test-suite.log')
raise UnexpectedExit(res)
+@task
+def ci_make_distdir(c):
+ res = c.run('make distdir')
+
@task
def ci_make_install(c):
res = c.run('make install') # FIXME: this builds auth docs - again
@task
-def add_auth_repo(c):
- dist = 'ubuntu' # FIXME take these from the caller?
- release = 'focal'
- version = '44'
-
- c.sudo('apt-get install -qq -y curl gnupg2')
- if version == 'master':
+def add_auth_repo(c, dist_name, dist_release_name, pdns_repo_version):
+ c.sudo('apt-get install -y curl gnupg2')
+ if pdns_repo_version == 'master':
c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc')
else:
c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc')
- c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist} {release}-auth-{version} main' | sudo tee /etc/apt/sources.list.d/pdns.list")
+ c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist_name} {dist_release_name}-auth-{pdns_repo_version} main' | sudo tee /etc/apt/sources.list.d/pdns.list")
c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns")
c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns")
c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns")
c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}')
elif product == 'auth':
with c.cd('regression-tests.api'):
- c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests authoritative {backend}')
+ c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432 ./runtests authoritative {backend}')
else:
raise Failure('unknown product')
godbc_mssql_credentials = {"username": "sa", "password": "SAsa12%%"}
-godbc_config = '''
+godbc_config = f'''
[pdns-mssql-docker]
Driver=FreeTDS
Trace=No
-Server=127.0.0.1
+Server={auth_backend_ip_addr}
Port=1433
Database=pdns
TDS_Version=7.1
[pdns-mssql-docker-nodb]
Driver=FreeTDS
Trace=No
-Server=127.0.0.1
+Server={auth_backend_ip_addr}
Port=1433
TDS_Version=7.1
c.sudo('sed -i "s/libsqlite3odbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libsqlite3odbc.so/g" /etc/odbcinst.ini')
def setup_ldap_client(c):
- c.sudo('DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ldap-utils')
- c.sudo('sh -c \'echo "127.0.0.1 ldapserver" | tee -a /etc/hosts\'')
+ c.sudo('DEBIAN_FRONTEND=noninteractive apt-get install -y ldap-utils')
+ c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} ldapserver" | tee -a /etc/hosts\'')
@task
def test_auth_backend(c, backend):
- pdns_auth_env_vars = 'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432"'
+ pdns_auth_env_vars = f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST={auth_backend_ip_addr} GMYSQL2HOST={auth_backend_ip_addr} MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432'
if backend == 'remote':
ci_auth_install_remotebackend_test_deps(c)
if backend == 'authpy':
+ c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} kerberos-server" | tee -a /etc/hosts\'')
with c.cd('regression-tests.auth-py'):
c.run(f'{pdns_auth_env_vars} WITHKERBEROS=YES ./runtests')
return
c.run(f'{pdns_auth_env_vars} ./start-test-stop 5300 {variant}')
if backend == 'gsqlite3':
+ if os.getenv('SKIP_IPV6_TESTS'):
+ pdns_auth_env_vars += ' context=noipv6'
with c.cd('regression-tests.nobackend'):
c.run(f'{pdns_auth_env_vars} ./runtests')
c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms')
@task
def test_regression_recursor(c):
c.run('/opt/pdns-recursor/sbin/pdns_recursor --version')
- c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control SKIP_IPV6_TESTS=y ./build-scripts/test-recursor')
+ c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control ./build-scripts/test-recursor')
@task
def test_bulk_recursor(c, threads, mthreads, shards):
@task
def coverity_clang_configure(c):
- c.sudo('/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-12')
+ c.sudo(f'/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-{clang_version}')
@task
def coverity_make(c):
projects / thirdparty / pdns.git / blobdiff