tasks.py

   1 from invoke import task
   2 from invoke.exceptions import Failure, UnexpectedExit
   3
   4 import os
   5 import sys
   6 import time
   7
   8 all_build_deps = [
   9     'ccache',
  10     'libboost-all-dev',
  11     'libluajit-5.1-dev',
  12     'libsodium-dev',
  13     'libssl-dev',
  14     'libsystemd-dev',
  15     'libtool',
  16     'make',
  17     'pkg-config',
  18     'python3-venv',
  19     'systemd',
  20 ]
  21 git_build_deps = [
  22     'autoconf',
  23     'automake',
  24     'bison',
  25     'bzip2',
  26     'curl',
  27     'flex',
  28     'git',
  29     'ragel'
  30 ]
  31 auth_build_deps = [    # FIXME: perhaps we should be stealing these from the debian (Ubuntu) control file
  32     'default-libmysqlclient-dev',
  33     'libcdb-dev',
  34     'libcurl4-openssl-dev',
  35     'libgeoip-dev',
  36     'libkrb5-dev',
  37     'libldap2-dev',
  38     'liblmdb-dev',
  39     'libmaxminddb-dev',
  40     'libp11-kit-dev',
  41     'libpq-dev',
  42     'libsqlite3-dev',
  43     'libyaml-cpp-dev',
  44     'libzmq3-dev',
  45     'ruby-bundler',
  46     'ruby-dev',
  47     'sqlite3',
  48     'unixodbc-dev',
  49 ]
  50 rec_build_deps = [
  51     'libcap-dev',
  52     'libfstrm-dev',
  53     'libsnmp-dev',
  54 ]
  55 rec_bulk_deps = [
  56     'curl',
  57     'libboost-all-dev',
  58     'libcap2',
  59     'libfstrm0',
  60     'libluajit-5.1-2',
  61     'libsnmp35',
  62     'libsodium23',
  63     'libssl1.1',
  64     'libsystemd0',
  65     'moreutils',
  66     'pdns-tools',
  67     'unzip',
  68 ]
  69 dnsdist_build_deps = [
  70     'libcap-dev',
  71     'libcdb-dev',
  72     'libedit-dev',
  73     'libfstrm-dev',
  74     'libgnutls28-dev',
  75     'libh2o-evloop-dev',
  76     'liblmdb-dev',
  77     'libnghttp2-dev',
  78     'libre2-dev',
  79     'libsnmp-dev',
  80 ]
  81 auth_test_deps = [   # FIXME: we should be generating some of these from shlibdeps in build
  82     'authbind',
  83     'bc',
  84     'bind9utils',
  85     'curl',
  86     'default-jre-headless',
  87     'dnsutils',
  88     'docker-compose',
  89     'faketime',
  90     'gawk',
  91     'krb5-user',
  92     'ldnsutils',
  93     'libboost-serialization1.71.0',
  94     'libcdb1',
  95     'libcurl4',
  96     'libgeoip1',
  97     'libkrb5-3',
  98     'libldap-2.4-2',
  99     'liblmdb0',
 100     'libluajit-5.1-2',
 101     'libmaxminddb0',
 102     'libnet-dns-perl',
 103     'libp11-kit0',
 104     'libpq5',
 105     'libsodium23',
 106     'libsqlite3-dev',
 107     'libssl1.1',
 108     'libsystemd0',
 109     'libyaml-cpp0.6',
 110     'libzmq3-dev',
 111     'prometheus',
 112     'ruby-bundler',
 113     'ruby-dev',
 114     'socat',
 115     'softhsm2',
 116     'unbound-host',
 117     'unixodbc',
 118     'wget',
 119 ]
 120 doc_deps = [
 121     'autoconf',
 122     'automake',
 123     'bison',
 124     'curl',
 125     'flex',
 126     'g++',
 127     'git',
 128     'latexmk',
 129     'libboost-all-dev',
 130     'libedit-dev',
 131     'libluajit-5.1-dev',
 132     'libssl-dev',
 133     'make',
 134     'pkg-config',
 135     'python3-venv',
 136     'ragel',
 137     'rsync',
 138 ]
 139 doc_deps_pdf = [
 140     'texlive-binaries',
 141     'texlive-formats-extra',
 142     'texlive-latex-extra',
 143 ]
 144
 145 @task
 146 def apt_fresh(c):
 147     c.sudo('sed -i \'s/azure\.//\' /etc/apt/sources.list')
 148     c.sudo('apt-get update')
 149     c.sudo('apt-get -qq -y --allow-downgrades dist-upgrade')
 150
 151 @task
 152 def install_clang(c):
 153     """
 154     install clang-12 and llvm-12
 155     """
 156     c.sudo('apt-get -qq -y --no-install-recommends install clang-12 llvm-12')
 157
 158 @task
 159 def install_clang_runtime(c):
 160     # this gives us the symbolizer, for symbols in asan/ubsan traces
 161     c.sudo('apt-get -qq -y --no-install-recommends install clang-12')
 162
 163 def install_libdecaf(c, product):
 164     c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf')
 165     with c.cd('/tmp/libdecaf'):
 166         c.run('git checkout 41f349')
 167         c.run('cmake -B build '
 168               '-DCMAKE_INSTALL_PREFIX=/usr/local '
 169               '-DCMAKE_INSTALL_LIBDIR=lib '
 170               '-DENABLE_STATIC=OFF '
 171               '-DENABLE_TESTS=OFF '
 172               '-DCMAKE_C_FLAGS="-Wno-sizeof-array-div -Wno-array-parameter" .')
 173         c.run('make -C build')
 174         c.run('sudo make -C build install')
 175     c.sudo(f'mkdir -p /opt/{product}/libdecaf')
 176     c.sudo(f'cp /usr/local/lib/libdecaf.so* /opt/{product}/libdecaf/.')
 177
 178 @task
 179 def install_doc_deps(c):
 180     c.sudo('apt-get install -qq -y ' + ' '.join(doc_deps))
 181
 182 @task
 183 def install_doc_deps_pdf(c):
 184     c.sudo('apt-get install -qq -y ' + ' '.join(doc_deps_pdf))
 185
 186 @task
 187 def install_auth_build_deps(c):
 188     c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps))
 189     install_libdecaf(c, 'pdns-auth')
 190
 191 def setup_authbind(c):
 192     c.sudo('touch /etc/authbind/byport/53')
 193     c.sudo('chmod 755 /etc/authbind/byport/53')
 194
 195 auth_backend_test_deps = dict(
 196     gsqlite3=['sqlite3'],
 197     gmysql=['default-libmysqlclient-dev'],
 198     gpgsql=['libpq-dev'],
 199     lmdb=[],
 200     remote=[],
 201     bind=[],
 202     geoip=[],
 203     lua2=[],
 204     tinydns=[],
 205     authpy=[],
 206     godbc_sqlite3=['libsqliteodbc'],
 207     godbc_mssql=['freetds-bin','tdsodbc'],
 208     ldap=[],
 209     geoip_mmdb=[]
 210 )
 211
 212 @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend'])
 213 def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more than apt-get
 214     extra=[]
 215     for b in backend:
 216         extra.extend(auth_backend_test_deps[b])
 217     c.sudo('apt-get -y -qq install ' + ' '.join(extra+auth_test_deps))
 218
 219     c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*')
 220     # c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then
 221     #               wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz
 222     #               tar xfz jdnssec-tools-0.14.tar.gz -C $HOME
 223     #               rm jdnssec-tools-0.14.tar.gz
 224     #          fi
 225     #          echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''')  # FIXME: why did this fail with no error?
 226     c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing regression-tests.nobackend/rectify-axfr/allow-missing') # FIXME: can this go?
 227     # FIXME we may want to start a background recursor here to make ALIAS tests more robust
 228     setup_authbind(c)
 229
 230     # Copy libdecaf out
 231     c.sudo('mkdir -p /usr/local/lib')
 232     c.sudo('cp /opt/pdns-auth/libdecaf/libdecaf.so* /usr/local/lib/.')
 233
 234 @task
 235 def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get
 236     c.sudo('apt-get --no-install-recommends -qq -y install ' + ' '.join(rec_bulk_deps))
 237     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 238
 239 @task
 240 def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get
 241     c.sudo('apt-get --no-install-recommends install -qq -y ' + ' '.join(rec_bulk_deps) + ' \
 242               pdns-server pdns-backend-bind daemontools \
 243               jq libfaketime lua-posix lua-socket bc authbind \
 244               python3-venv python3-dev default-libmysqlclient-dev libpq-dev \
 245               protobuf-compiler snmpd prometheus')
 246
 247     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 248
 249     setup_authbind(c)
 250
 251     c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 252     c.sudo('systemctl restart snmpd')
 253     time.sleep(5)
 254     c.sudo('chmod 755 /var/agentx')
 255
 256 @task
 257 def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt-get
 258     c.sudo('apt-get install -qq -y \
 259               libluajit-5.1-2 \
 260               libboost-all-dev \
 261               libcap2 \
 262               libcdb1 \
 263               libcurl4-openssl-dev \
 264               libfstrm0 \
 265               libgnutls30 \
 266               libh2o-evloop0.13 \
 267               liblmdb0 \
 268               libnghttp2-14 \
 269               libre2-5 \
 270               libssl-dev \
 271               libsystemd0 \
 272               libsodium23 \
 273               lua-socket \
 274               patch \
 275               protobuf-compiler \
 276               python3-venv snmpd prometheus')
 277     c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 278     c.sudo('systemctl restart snmpd')
 279     time.sleep(5)
 280     c.sudo('chmod 755 /var/agentx')
 281
 282 @task
 283 def install_rec_build_deps(c):
 284     c.sudo('apt-get install -qq -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + rec_build_deps))
 285
 286 @task
 287 def install_dnsdist_build_deps(c):
 288     c.sudo('apt-get install -qq -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps))
 289
 290 @task
 291 def ci_autoconf(c):
 292     c.run('BUILDER_VERSION=0.0.0-git1 autoreconf -vfi')
 293
 294 @task
 295 def ci_docs_build(c):
 296     c.run('make -f Makefile.sphinx -C docs html')
 297
 298 @task
 299 def ci_docs_build_pdf(c):
 300     c.run('make -f Makefile.sphinx -C docs latexpdf')
 301
 302 @task
 303 def ci_docs_upload_master(c, docs_host, pdf, username, product, directory=""):
 304     c.run(f"rsync -crv --delete --no-p --chmod=g=rwX --exclude '*~' ./docs/_build/{product}-html-docs/ {username}@{docs_host}:{directory}")
 305     c.run(f"rsync -crv --no-p --chmod=g=rwX --exclude '*~' ./docs/_build/{product}-html-docs.tar.bz2 {username}@{docs_host}:{directory}/html-docs.tar.bz2")
 306     c.run(f"rsync -crv --no-p --chmod=g=rwX --exclude '*~' ./docs/_build/latex/{pdf} {username}@{docs_host}:{directory}")
 307
 308 @task
 309 def ci_docs_add_ssh(c, ssh_key, host_key):
 310     c.run('mkdir -m 700 -p ~/.ssh')
 311     c.run(f'echo "{ssh_key}" > ~/.ssh/id_ed25519')
 312     c.run('chmod 600 ~/.ssh/id_ed25519')
 313     c.run(f'echo "{host_key}" > ~/.ssh/known_hosts')
 314
 315 @task
 316 def ci_auth_configure(c):
 317     sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
 318     unittests = ' --enable-unit-tests --enable-backend-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
 319     fuzzingtargets = ' --enable-fuzz-targets' if os.getenv('FUZZING_TARGETS') == 'yes' else ''
 320     res = c.run('''CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
 321                    CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
 322                    ./configure \
 323                       CC='clang-12' \
 324                       CXX='clang++-12' \
 325                       LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib' \
 326                       --enable-option-checking=fatal \
 327                       --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' \
 328                       --enable-systemd \
 329                       --enable-tools \
 330                       --enable-fuzz-targets \
 331                       --enable-experimental-pkcs11 \
 332                       --enable-experimental-gss-tsig \
 333                       --enable-remotebackend-zeromq \
 334                       --with-lmdb=/usr \
 335                       --with-libsodium \
 336                       --with-libdecaf \
 337                       --prefix=/opt/pdns-auth \
 338                       --enable-ixfrdist \
 339                       --enable-fortify-source=auto \
 340                       --enable-auto-var-init=pattern ''' + sanitizers + unittests + fuzzingtargets, warn=True)
 341     if res.exited != 0:
 342         c.run('cat config.log')
 343         raise UnexpectedExit(res)
 344 @task
 345 def ci_rec_configure(c):
 346     sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
 347     unittests = ' --enable-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
 348     res = c.run('''            CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
 349             CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
 350             ./configure \
 351               CC='clang-12' \
 352               CXX='clang++-12' \
 353               --enable-option-checking=fatal \
 354               --enable-nod \
 355               --enable-systemd \
 356               --prefix=/opt/pdns-recursor \
 357               --with-libsodium \
 358               --with-lua=luajit \
 359               --with-libcap \
 360               --with-net-snmp \
 361               --enable-fortify-source=auto \
 362               --enable-auto-var-init=pattern \
 363               --enable-dns-over-tls ''' + sanitizers + unittests, warn=True)
 364     if res.exited != 0:
 365         c.run('cat config.log')
 366         raise UnexpectedExit(res)
 367
 368 @task
 369 def ci_dnsdist_configure(c, features):
 370     additional_flags = ''
 371     if features == 'full':
 372       features_set = '--enable-dnstap \
 373                       --enable-dnscrypt \
 374                       --enable-dns-over-tls \
 375                       --enable-dns-over-https \
 376                       --enable-systemd \
 377                       --prefix=/opt/dnsdist \
 378                       --with-gnutls \
 379                       --with-libsodium \
 380                       --with-lua=luajit \
 381                       --with-libcap \
 382                       --with-nghttp2 \
 383                       --with-re2 '
 384     else:
 385       features_set = '--disable-dnstap \
 386                       --disable-dnscrypt \
 387                       --disable-ipcipher \
 388                       --disable-systemd \
 389                       --without-cdb \
 390                       --without-ebpf \
 391                       --without-gnutls \
 392                       --without-libedit \
 393                       --without-libsodium \
 394                       --without-lmdb \
 395                       --without-net-snmp \
 396                       --without-nghttp2 \
 397                       --without-re2 '
 398       additional_flags = '-DDISABLE_COMPLETION \
 399                           -DDISABLE_DELAY_PIPE \
 400                           -DDISABLE_DYNBLOCKS \
 401                           -DDISABLE_PROMETHEUS \
 402                           -DDISABLE_PROTOBUF \
 403                           -DDISABLE_BUILTIN_HTML \
 404                           -DDISABLE_CARBON \
 405                           -DDISABLE_SECPOLL \
 406                           -DDISABLE_DEPRECATED_DYNBLOCK \
 407                           -DDISABLE_LUA_WEB_HANDLERS \
 408                           -DDISABLE_NON_FFI_DQ_BINDINGS \
 409                           -DDISABLE_POLICIES_BINDINGS \
 410                           -DDISABLE_PACKETCACHE_BINDINGS \
 411                           -DDISABLE_DOWNSTREAM_BINDINGS \
 412                           -DDISABLE_COMBO_ADDR_BINDINGS \
 413                           -DDISABLE_CLIENT_STATE_BINDINGS \
 414                           -DDISABLE_QPS_LIMITER_BINDINGS \
 415                           -DDISABLE_SUFFIX_MATCH_BINDINGS \
 416                           -DDISABLE_NETMASK_BINDINGS \
 417                           -DDISABLE_DNSNAME_BINDINGS \
 418                           -DDISABLE_DNSHEADER_BINDINGS \
 419                           -DDISABLE_RECVMMSG \
 420                           -DDISABLE_WEB_CACHE_MANAGEMENT \
 421                           -DDISABLE_WEB_CONFIG \
 422                           -DDISABLE_RULES_ALTERING_QUERIES \
 423                           -DDISABLE_ECS_ACTIONS \
 424                           -DDISABLE_TOP_N_BINDINGS \
 425                           -DDISABLE_OCSP_STAPLING \
 426                           -DDISABLE_HASHED_CREDENTIALS \
 427                           -DDISABLE_FALSE_SHARING_PADDING \
 428                           -DDISABLE_NPN'
 429     unittests = ' --enable-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
 430     sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
 431     cflags = '-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int'
 432     cxxflags = cflags + ' -Wp,-D_GLIBCXX_ASSERTIONS ' + additional_flags
 433     res = c.run('''CFLAGS="%s" \
 434                    CXXFLAGS="%s" \
 435                    AR=llvm-ar-12 \
 436                    RANLIB=llvm-ranlib-12 \
 437                    ./configure \
 438                      CC='clang-12' \
 439                      CXX='clang++-12' \
 440                      --enable-option-checking=fatal \
 441                      --enable-fortify-source=auto \
 442                      --enable-auto-var-init=pattern \
 443                      --enable-lto=thin \
 444                      --prefix=/opt/dnsdist %s %s %s''' % (cflags, cxxflags, features_set, sanitizers, unittests), warn=True)
 445     if res.exited != 0:
 446         c.run('cat config.log')
 447         raise UnexpectedExit(res)
 448
 449 @task
 450 def ci_auth_make(c):
 451     c.run('make -j8 -k V=1')
 452
 453 @task
 454 def ci_rec_make(c):
 455     c.run('make -j8 -k V=1')
 456
 457 @task
 458 def ci_dnsdist_make(c):
 459     c.run('make -j4 -k V=1')
 460
 461 @task
 462 def ci_auth_install_remotebackend_test_deps(c):
 463     with c.cd('modules/remotebackend'):
 464       # c.run('bundle config set path vendor/bundle')
 465       c.run('sudo ruby -S bundle install')
 466     c.sudo('apt-get install -qq -y socat')
 467
 468 @task
 469 def ci_auth_run_unit_tests(c):
 470     res = c.run('make check', warn=True)
 471     if res.exited != 0:
 472       c.run('cat pdns/test-suite.log', warn=True)
 473       c.run('cat modules/remotebackend/test-suite.log', warn=True)
 474       raise UnexpectedExit(res)
 475
 476 @task
 477 def ci_rec_run_unit_tests(c):
 478     res = c.run('make check', warn=True)
 479     if res.exited != 0:
 480       c.run('cat test-suite.log')
 481       raise UnexpectedExit(res)
 482
 483 @task
 484 def ci_dnsdist_run_unit_tests(c):
 485     res = c.run('make check', warn=True)
 486     if res.exited != 0:
 487       c.run('cat test-suite.log')
 488       raise UnexpectedExit(res)
 489
 490 @task
 491 def ci_make_install(c):
 492     res = c.run('make install') # FIXME: this builds auth docs - again
 493
 494 @task
 495 def add_auth_repo(c):
 496     dist = 'ubuntu' # FIXME take these from the caller?
 497     release = 'focal'
 498     version = '44'
 499
 500     c.sudo('apt-get install -qq -y curl gnupg2')
 501     if version == 'master':
 502         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc')
 503     else:
 504         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc')
 505     c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist} {release}-auth-{version} main' | sudo tee /etc/apt/sources.list.d/pdns.list")
 506     c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns")
 507     c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns")
 508     c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns")
 509     c.sudo('apt-get update')
 510
 511 @task
 512 def test_api(c, product, backend=''):
 513     if product == 'recursor':
 514         with c.cd('regression-tests.api'):
 515             c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}')
 516     elif product == 'auth':
 517         with c.cd('regression-tests.api'):
 518             c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests authoritative {backend}')
 519     else:
 520         raise Failure('unknown product')
 521
 522 backend_regress_tests = dict(
 523     bind = [
 524         'bind-both',
 525         'bind-dnssec-both',
 526         'bind-dnssec-nsec3-both',
 527         'bind-dnssec-nsec3-optout-both',
 528         'bind-dnssec-nsec3-narrow',
 529         # FIXME  'bind-dnssec-pkcs11'
 530     ],
 531     geoip = [
 532         'geoip',
 533         'geoip-nsec3-narrow'
 534     ],
 535     lua2 = ['lua2', 'lua2-dnssec'],
 536     tinydns = ['tinydns'],
 537     remote = [
 538         'remotebackend-pipe',
 539         'remotebackend-unix',
 540         'remotebackend-http',
 541         'remotebackend-zeromq',
 542         'remotebackend-pipe-dnssec',
 543         'remotebackend-unix-dnssec',
 544         'remotebackend-http-dnssec',
 545         'remotebackend-zeromq-dnssec'
 546     ],
 547     lmdb = [
 548         'lmdb-nodnssec-both',
 549         'lmdb-both',
 550         'lmdb-nsec3-both',
 551         'lmdb-nsec3-optout-both',
 552         'lmdb-nsec3-narrow'
 553     ],
 554     gmysql = [
 555         'gmysql',
 556         'gmysql-nodnssec-both',
 557         'gmysql-nsec3-both',
 558         'gmysql-nsec3-optout-both',
 559         'gmysql-nsec3-narrow',
 560         'gmysql_sp-both'
 561     ],
 562     gpgsql = [
 563         'gpgsql',
 564         'gpgsql-nodnssec-both',
 565         'gpgsql-nsec3-both',
 566         'gpgsql-nsec3-optout-both',
 567         'gpgsql-nsec3-narrow',
 568         'gpgsql_sp-both'
 569     ],
 570     gsqlite3 = [
 571         'gsqlite3',
 572         'gsqlite3-nodnssec-both',
 573         'gsqlite3-nsec3-both',
 574         'gsqlite3-nsec3-optout-both',
 575         'gsqlite3-nsec3-narrow'
 576     ],
 577     godbc_sqlite3 = ['godbc_sqlite3-nodnssec'],
 578     godbc_mssql = [
 579         'godbc_mssql',
 580         'godbc_mssql-nodnssec',
 581         'godbc_mssql-nsec3',
 582         'godbc_mssql-nsec3-optout',
 583         'godbc_mssql-nsec3-narrow'
 584     ],
 585     ldap = [
 586         'ldap-tree',
 587         'ldap-simple',
 588         'ldap-strict'
 589     ],
 590     geoip_mmdb = ['geoip'],
 591 )
 592
 593 godbc_mssql_credentials = {"username": "sa", "password": "SAsa12%%"}
 594
 595 godbc_config = '''
 596 [pdns-mssql-docker]
 597 Driver=FreeTDS
 598 Trace=No
 599 Server=127.0.0.1
 600 Port=1433
 601 Database=pdns
 602 TDS_Version=7.1
 603
 604 [pdns-mssql-docker-nodb]
 605 Driver=FreeTDS
 606 Trace=No
 607 Server=127.0.0.1
 608 Port=1433
 609 TDS_Version=7.1
 610
 611 [pdns-sqlite3-1]
 612 Driver = SQLite3
 613 Database = pdns.sqlite3
 614
 615 [pdns-sqlite3-2]
 616 Driver = SQLite3
 617 Database = pdns.sqlite32
 618 '''
 619
 620 def setup_godbc_mssql(c):
 621     with open(os.path.expanduser("~/.odbc.ini"), "a") as f:
 622         f.write(godbc_config)
 623     c.sudo('sh -c \'echo "Threading=1" | cat /usr/share/tdsodbc/odbcinst.ini - | tee -a /etc/odbcinst.ini\'')
 624     c.sudo('sed -i "s/libtdsodbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libtdsodbc.so/g" /etc/odbcinst.ini')
 625     c.run(f'echo "create database pdns" | isql -v pdns-mssql-docker-nodb {godbc_mssql_credentials["username"]} {godbc_mssql_credentials["password"]}')
 626     # FIXME: Skip 8bit-txt-unescaped test
 627     c.run('touch ${PWD}/regression-tests/tests/8bit-txt-unescaped/skip')
 628
 629 def setup_godbc_sqlite3(c):
 630     with open(os.path.expanduser("~/.odbc.ini"), "a") as f:
 631         f.write(godbc_config)
 632     c.sudo('sed -i "s/libsqlite3odbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libsqlite3odbc.so/g" /etc/odbcinst.ini')
 633
 634 def setup_ldap_client(c):
 635     c.sudo('DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ldap-utils')
 636     c.sudo('sh -c \'echo "127.0.0.1 ldapserver" | tee -a /etc/hosts\'')
 637
 638 @task
 639 def test_auth_backend(c, backend):
 640     pdns_auth_env_vars = 'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432"'
 641
 642     if backend == 'remote':
 643         ci_auth_install_remotebackend_test_deps(c)
 644
 645     if backend == 'authpy':
 646         with c.cd('regression-tests.auth-py'):
 647             c.run(f'{pdns_auth_env_vars} WITHKERBEROS=YES ./runtests')
 648         return
 649
 650     if backend == 'godbc_sqlite3':
 651         setup_godbc_sqlite3(c)
 652         with c.cd('regression-tests'):
 653             for variant in backend_regress_tests[backend]:
 654                 c.run(f'{pdns_auth_env_vars} GODBC_SQLITE3_DSN=pdns-sqlite3-1 ./start-test-stop 5300 {variant}')
 655         return
 656
 657     if backend == 'godbc_mssql':
 658         setup_godbc_mssql(c)
 659         with c.cd('regression-tests'):
 660             for variant in backend_regress_tests[backend]:
 661                 c.run(f'{pdns_auth_env_vars} GODBC_MSSQL_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL_DSN=pdns-mssql-docker GODBC_MSSQL2_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL2_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL2_DSN=pdns-mssql-docker ./start-test-stop 5300 {variant}')
 662         return
 663
 664     if backend == 'ldap':
 665         setup_ldap_client(c)
 666
 667     if backend == 'geoip_mmdb':
 668         with c.cd('regression-tests'):
 669             for variant in backend_regress_tests[backend]:
 670                 c.run(f'{pdns_auth_env_vars} geoipdatabase=../modules/geoipbackend/regression-tests/GeoLiteCity.mmdb ./start-test-stop 5300 {variant}')
 671         return
 672
 673     with c.cd('regression-tests'):
 674         if backend == 'lua2':
 675             c.run('touch trustedkeys')  # avoid silly error during cleanup
 676         for variant in backend_regress_tests[backend]:
 677             c.run(f'{pdns_auth_env_vars} ./start-test-stop 5300 {variant}')
 678
 679     if backend == 'gsqlite3':
 680         with c.cd('regression-tests.nobackend'):
 681             c.run(f'{pdns_auth_env_vars} ./runtests')
 682         c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms')
 683         return
 684
 685 @task
 686 def test_ixfrdist(c):
 687     with c.cd('regression-tests.ixfrdist'):
 688         c.run('IXFRDISTBIN=/opt/pdns-auth/bin/ixfrdist ./runtests')
 689
 690 @task
 691 def test_dnsdist(c):
 692     c.run('chmod +x /opt/dnsdist/bin/*')
 693     c.run('ls -ald /var /var/agentx /var/agentx/master')
 694     c.run('ls -al /var/agentx/master')
 695     with c.cd('regression-tests.dnsdist'):
 696         c.run('DNSDISTBIN=/opt/dnsdist/bin/dnsdist ./runtests')
 697
 698 @task
 699 def test_regression_recursor(c):
 700     c.run('/opt/pdns-recursor/sbin/pdns_recursor --version')
 701     c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control SKIP_IPV6_TESTS=y ./build-scripts/test-recursor')
 702
 703 @task
 704 def test_bulk_recursor(c, threads, mthreads, shards):
 705     # We run an extremely small version of the bulk test, as GH does not seem to be able to handle the UDP load
 706     with c.cd('regression-tests'):
 707         c.run('curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip')
 708         c.run('unzip top-1m.csv.zip -d .')
 709         c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 710         c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control THRESHOLD=95 TRACE=no ./timestamp ./recursor-test 5300 100 {threads} {mthreads} {shards}')
 711
 712 @task
 713 def install_swagger_tools(c):
 714     c.run('npm install -g api-spec-converter')
 715
 716 @task
 717 def swagger_syntax_check(c):
 718     c.run('api-spec-converter docs/http-api/swagger/authoritative-api-swagger.yaml -f swagger_2 -t openapi_3 -s json -c')
 719
 720 @task
 721 def install_coverity_tools(c, project):
 722     token = os.getenv('COVERITY_TOKEN')
 723     c.run(f'curl -s https://scan.coverity.com/download/linux64 --data "token={token}&project={project}" | gunzip | sudo tar xvf /dev/stdin --strip-components=1 --no-same-owner -C /usr/local', hide=True)
 724
 725 @task
 726 def coverity_clang_configure(c):
 727     c.sudo('/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-12')
 728
 729 @task
 730 def coverity_make(c):
 731     c.run('/usr/local/bin/cov-build --dir cov-int make -j8 -k')
 732
 733 @task
 734 def coverity_tarball(c, tarball):
 735     c.run(f'tar caf {tarball} cov-int')
 736
 737 @task
 738 def coverity_upload(c, email, project, tarball):
 739     token = os.getenv('COVERITY_TOKEN')
 740     c.run(f'curl --form token={token} \
 741             --form email="{email}" \
 742             --form file=@{tarball} \
 743             --form version="$(./builder-support/gen-version)" \
 744             --form description="master build" \
 745             https://scan.coverity.com/builds?project={project}', hide=True)
 746
 747 # this is run always
 748 def setup():
 749     if '/usr/lib/ccache' not in os.environ['PATH']:
 750         os.environ['PATH']='/usr/lib/ccache:'+os.environ['PATH']
 751
 752 setup()