tasks.py

   1 from invoke import task
   2 from invoke.exceptions import Failure, UnexpectedExit
   3
   4 import os
   5 import sys
   6 import time
   7
   8 all_build_deps = [
   9     'ccache',
  10     'libboost-all-dev',
  11     'libluajit-5.1-dev',
  12     'libsodium-dev',
  13     'libssl-dev',
  14     'libsystemd-dev',
  15     'libtool',
  16     'make',
  17     'pkg-config',
  18     'python3-venv',
  19     'systemd',
  20 ]
  21 git_build_deps = [
  22     'autoconf',
  23     'automake',
  24     'bison',
  25     'bzip2',
  26     'curl',
  27     'flex',
  28     'git',
  29     'ragel'
  30 ]
  31 auth_build_deps = [    # FIXME: perhaps we should be stealing these from the debian (Ubuntu) control file
  32     'default-libmysqlclient-dev',
  33     'libcdb-dev',
  34     'libcurl4-openssl-dev',
  35     'libgeoip-dev',
  36     'libkrb5-dev',
  37     'libldap2-dev',
  38     'liblmdb-dev',
  39     'libmaxminddb-dev',
  40     'libp11-kit-dev',
  41     'libpq-dev',
  42     'libsqlite3-dev',
  43     'libyaml-cpp-dev',
  44     'libzmq3-dev',
  45     'ruby-bundler',
  46     'ruby-dev',
  47     'sqlite3',
  48     'unixodbc-dev',
  49 ]
  50 rec_build_deps = [
  51     'libcap-dev',
  52     'libfstrm-dev',
  53     'libsnmp-dev',
  54 ]
  55 rec_bulk_deps = [
  56     'curl',
  57     'libboost-all-dev',
  58     'libcap2',
  59     'libfstrm0',
  60     'libluajit-5.1-2',
  61     'libsnmp35',
  62     'libsodium23',
  63     'libssl1.1',
  64     'libsystemd0',
  65     'moreutils',
  66     'pdns-tools',
  67     'unzip',
  68 ]
  69 dnsdist_build_deps = [
  70     'libcap-dev',
  71     'libcdb-dev',
  72     'libedit-dev',
  73     'libfstrm-dev',
  74     'libgnutls28-dev',
  75     'libh2o-evloop-dev',
  76     'liblmdb-dev',
  77     'libnghttp2-dev',
  78     'libre2-dev',
  79     'libsnmp-dev',
  80 ]
  81 auth_test_deps = [   # FIXME: we should be generating some of these from shlibdeps in build
  82     'authbind',
  83     'bc',
  84     'bind9utils',
  85     'curl',
  86     'default-jre-headless',
  87     'dnsutils',
  88     'docker-compose',
  89     'faketime',
  90     'gawk',
  91     'krb5-user',
  92     'ldnsutils',
  93     'libboost-serialization1.71.0',
  94     'libcdb1',
  95     'libcurl4',
  96     'libgeoip1',
  97     'libkrb5-3',
  98     'libldap-2.4-2',
  99     'liblmdb0',
 100     'libluajit-5.1-2',
 101     'libmaxminddb0',
 102     'libnet-dns-perl',
 103     'libp11-kit0',
 104     'libpq5',
 105     'libsodium23',
 106     'libsqlite3-dev',
 107     'libssl1.1',
 108     'libsystemd0',
 109     'libyaml-cpp0.6',
 110     'libzmq3-dev',
 111     'lmdb-utils',
 112     'prometheus',
 113     'ruby-bundler',
 114     'ruby-dev',
 115     'socat',
 116     'softhsm2',
 117     'unbound-host',
 118     'unixodbc',
 119     'wget',
 120 ]
 121 doc_deps = [
 122     'autoconf',
 123     'automake',
 124     'bison',
 125     'curl',
 126     'flex',
 127     'g++',
 128     'git',
 129     'latexmk',
 130     'libboost-all-dev',
 131     'libedit-dev',
 132     'libluajit-5.1-dev',
 133     'libssl-dev',
 134     'make',
 135     'pkg-config',
 136     'python3-venv',
 137     'ragel',
 138     'rsync',
 139 ]
 140 doc_deps_pdf = [
 141     'texlive-binaries',
 142     'texlive-formats-extra',
 143     'texlive-latex-extra',
 144 ]
 145
 146 @task
 147 def apt_fresh(c):
 148     c.sudo('sed -i \'s/azure\.//\' /etc/apt/sources.list')
 149     c.sudo('apt-get update')
 150     c.sudo('apt-get -qq -y --allow-downgrades dist-upgrade')
 151
 152 @task
 153 def install_clang(c):
 154     """
 155     install clang-12 and llvm-12
 156     """
 157     c.sudo('apt-get -qq -y --no-install-recommends install clang-12 llvm-12')
 158
 159 @task
 160 def install_clang_runtime(c):
 161     # this gives us the symbolizer, for symbols in asan/ubsan traces
 162     c.sudo('apt-get -qq -y --no-install-recommends install clang-12')
 163
 164 def install_libdecaf(c, product):
 165     c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf')
 166     with c.cd('/tmp/libdecaf'):
 167         c.run('git checkout 41f349')
 168         c.run('cmake -B build '
 169               '-DCMAKE_INSTALL_PREFIX=/usr/local '
 170               '-DCMAKE_INSTALL_LIBDIR=lib '
 171               '-DENABLE_STATIC=OFF '
 172               '-DENABLE_TESTS=OFF '
 173               '-DCMAKE_C_FLAGS="-Wno-sizeof-array-div -Wno-array-parameter" .')
 174         c.run('make -C build')
 175         c.run('sudo make -C build install')
 176     c.sudo(f'mkdir -p /opt/{product}/libdecaf')
 177     c.sudo(f'cp /usr/local/lib/libdecaf.so* /opt/{product}/libdecaf/.')
 178
 179 @task
 180 def install_doc_deps(c):
 181     c.sudo('apt-get install -qq -y ' + ' '.join(doc_deps))
 182
 183 @task
 184 def install_doc_deps_pdf(c):
 185     c.sudo('apt-get install -qq -y ' + ' '.join(doc_deps_pdf))
 186
 187 @task
 188 def install_auth_build_deps(c):
 189     c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps))
 190     install_libdecaf(c, 'pdns-auth')
 191
 192 def setup_authbind(c):
 193     c.sudo('touch /etc/authbind/byport/53')
 194     c.sudo('chmod 755 /etc/authbind/byport/53')
 195
 196 auth_backend_test_deps = dict(
 197     gsqlite3=['sqlite3'],
 198     gmysql=['default-libmysqlclient-dev'],
 199     gpgsql=['libpq-dev'],
 200     lmdb=[],
 201     remote=[],
 202     bind=[],
 203     geoip=[],
 204     lua2=[],
 205     tinydns=[],
 206     authpy=[],
 207     godbc_sqlite3=['libsqliteodbc'],
 208     godbc_mssql=['freetds-bin','tdsodbc'],
 209     ldap=[],
 210     geoip_mmdb=[]
 211 )
 212
 213 @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend'])
 214 def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more than apt-get
 215     extra=[]
 216     for b in backend:
 217         extra.extend(auth_backend_test_deps[b])
 218     c.sudo('apt-get -y -qq install ' + ' '.join(extra+auth_test_deps))
 219
 220     c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*')
 221     # c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then
 222     #               wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz
 223     #               tar xfz jdnssec-tools-0.14.tar.gz -C $HOME
 224     #               rm jdnssec-tools-0.14.tar.gz
 225     #          fi
 226     #          echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''')  # FIXME: why did this fail with no error?
 227     c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing regression-tests.nobackend/rectify-axfr/allow-missing') # FIXME: can this go?
 228     # FIXME we may want to start a background recursor here to make ALIAS tests more robust
 229     setup_authbind(c)
 230
 231     # Copy libdecaf out
 232     c.sudo('mkdir -p /usr/local/lib')
 233     c.sudo('cp /opt/pdns-auth/libdecaf/libdecaf.so* /usr/local/lib/.')
 234
 235 @task
 236 def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get
 237     c.sudo('apt-get --no-install-recommends -qq -y install ' + ' '.join(rec_bulk_deps))
 238     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 239
 240 @task
 241 def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get
 242     c.sudo('apt-get --no-install-recommends install -qq -y ' + ' '.join(rec_bulk_deps) + ' \
 243               pdns-server pdns-backend-bind daemontools \
 244               jq libfaketime lua-posix lua-socket bc authbind \
 245               python3-venv python3-dev default-libmysqlclient-dev libpq-dev \
 246               protobuf-compiler snmpd prometheus')
 247
 248     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 249
 250     setup_authbind(c)
 251
 252     c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 253     c.sudo('systemctl restart snmpd')
 254     time.sleep(5)
 255     c.sudo('chmod 755 /var/agentx')
 256
 257 @task
 258 def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt-get
 259     c.sudo('apt-get install -qq -y \
 260               libluajit-5.1-2 \
 261               libboost-all-dev \
 262               libcap2 \
 263               libcdb1 \
 264               libcurl4-openssl-dev \
 265               libfstrm0 \
 266               libgnutls30 \
 267               libh2o-evloop0.13 \
 268               liblmdb0 \
 269               libnghttp2-14 \
 270               libre2-5 \
 271               libssl-dev \
 272               libsystemd0 \
 273               libsodium23 \
 274               lua-socket \
 275               patch \
 276               protobuf-compiler \
 277               python3-venv snmpd prometheus')
 278     c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 279     c.sudo('systemctl restart snmpd')
 280     time.sleep(5)
 281     c.sudo('chmod 755 /var/agentx')
 282
 283 @task
 284 def install_rec_build_deps(c):
 285     c.sudo('apt-get install -qq -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + rec_build_deps))
 286
 287 @task
 288 def install_dnsdist_build_deps(c):
 289     c.sudo('apt-get install -qq -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps))
 290
 291 @task
 292 def ci_autoconf(c):
 293     c.run('BUILDER_VERSION=0.0.0-git1 autoreconf -vfi')
 294
 295 @task
 296 def ci_docs_build(c):
 297     c.run('make -f Makefile.sphinx -C docs html')
 298
 299 @task
 300 def ci_docs_build_pdf(c):
 301     c.run('make -f Makefile.sphinx -C docs latexpdf')
 302
 303 @task
 304 def ci_docs_upload_master(c, docs_host, pdf, username, product, directory=""):
 305     c.run(f"rsync -crv --delete --no-p --chmod=g=rwX --exclude '*~' ./docs/_build/{product}-html-docs/ {username}@{docs_host}:{directory}")
 306     c.run(f"rsync -crv --no-p --chmod=g=rwX --exclude '*~' ./docs/_build/{product}-html-docs.tar.bz2 {username}@{docs_host}:{directory}/html-docs.tar.bz2")
 307     c.run(f"rsync -crv --no-p --chmod=g=rwX --exclude '*~' ./docs/_build/latex/{pdf} {username}@{docs_host}:{directory}")
 308
 309 @task
 310 def ci_docs_add_ssh(c, ssh_key, host_key):
 311     c.run('mkdir -m 700 -p ~/.ssh')
 312     c.run(f'echo "{ssh_key}" > ~/.ssh/id_ed25519')
 313     c.run('chmod 600 ~/.ssh/id_ed25519')
 314     c.run(f'echo "{host_key}" > ~/.ssh/known_hosts')
 315
 316 @task
 317 def ci_auth_configure(c):
 318     sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
 319     unittests = ' --enable-unit-tests --enable-backend-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
 320     fuzzingtargets = ' --enable-fuzz-targets' if os.getenv('FUZZING_TARGETS') == 'yes' else ''
 321     res = c.run('''CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
 322                    CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
 323                    ./configure \
 324                       CC='clang-12' \
 325                       CXX='clang++-12' \
 326                       LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib' \
 327                       --enable-option-checking=fatal \
 328                       --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' \
 329                       --enable-systemd \
 330                       --enable-tools \
 331                       --enable-fuzz-targets \
 332                       --enable-experimental-pkcs11 \
 333                       --enable-experimental-gss-tsig \
 334                       --enable-remotebackend-zeromq \
 335                       --with-lmdb=/usr \
 336                       --with-libsodium \
 337                       --with-libdecaf \
 338                       --prefix=/opt/pdns-auth \
 339                       --enable-ixfrdist \
 340                       --enable-fortify-source=auto \
 341                       --enable-auto-var-init=pattern ''' + sanitizers + unittests + fuzzingtargets, warn=True)
 342     if res.exited != 0:
 343         c.run('cat config.log')
 344         raise UnexpectedExit(res)
 345 @task
 346 def ci_rec_configure(c):
 347     sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
 348     unittests = ' --enable-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
 349     res = c.run('''            CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
 350             CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
 351             ./configure \
 352               CC='clang-12' \
 353               CXX='clang++-12' \
 354               --enable-option-checking=fatal \
 355               --enable-nod \
 356               --enable-systemd \
 357               --prefix=/opt/pdns-recursor \
 358               --with-libsodium \
 359               --with-lua=luajit \
 360               --with-libcap \
 361               --with-net-snmp \
 362               --enable-fortify-source=auto \
 363               --enable-auto-var-init=pattern \
 364               --enable-dns-over-tls ''' + sanitizers + unittests, warn=True)
 365     if res.exited != 0:
 366         c.run('cat config.log')
 367         raise UnexpectedExit(res)
 368
 369 @task
 370 def ci_dnsdist_configure(c, features):
 371     additional_flags = ''
 372     if features == 'full':
 373       features_set = '--enable-dnstap \
 374                       --enable-dnscrypt \
 375                       --enable-dns-over-tls \
 376                       --enable-dns-over-https \
 377                       --enable-systemd \
 378                       --prefix=/opt/dnsdist \
 379                       --with-gnutls \
 380                       --with-libsodium \
 381                       --with-lua=luajit \
 382                       --with-libcap \
 383                       --with-nghttp2 \
 384                       --with-re2 '
 385     else:
 386       features_set = '--disable-dnstap \
 387                       --disable-dnscrypt \
 388                       --disable-ipcipher \
 389                       --disable-systemd \
 390                       --without-cdb \
 391                       --without-ebpf \
 392                       --without-gnutls \
 393                       --without-libedit \
 394                       --without-libsodium \
 395                       --without-lmdb \
 396                       --without-net-snmp \
 397                       --without-nghttp2 \
 398                       --without-re2 '
 399       additional_flags = '-DDISABLE_COMPLETION \
 400                           -DDISABLE_DELAY_PIPE \
 401                           -DDISABLE_DYNBLOCKS \
 402                           -DDISABLE_PROMETHEUS \
 403                           -DDISABLE_PROTOBUF \
 404                           -DDISABLE_BUILTIN_HTML \
 405                           -DDISABLE_CARBON \
 406                           -DDISABLE_SECPOLL \
 407                           -DDISABLE_DEPRECATED_DYNBLOCK \
 408                           -DDISABLE_LUA_WEB_HANDLERS \
 409                           -DDISABLE_NON_FFI_DQ_BINDINGS \
 410                           -DDISABLE_POLICIES_BINDINGS \
 411                           -DDISABLE_PACKETCACHE_BINDINGS \
 412                           -DDISABLE_DOWNSTREAM_BINDINGS \
 413                           -DDISABLE_COMBO_ADDR_BINDINGS \
 414                           -DDISABLE_CLIENT_STATE_BINDINGS \
 415                           -DDISABLE_QPS_LIMITER_BINDINGS \
 416                           -DDISABLE_SUFFIX_MATCH_BINDINGS \
 417                           -DDISABLE_NETMASK_BINDINGS \
 418                           -DDISABLE_DNSNAME_BINDINGS \
 419                           -DDISABLE_DNSHEADER_BINDINGS \
 420                           -DDISABLE_RECVMMSG \
 421                           -DDISABLE_WEB_CACHE_MANAGEMENT \
 422                           -DDISABLE_WEB_CONFIG \
 423                           -DDISABLE_RULES_ALTERING_QUERIES \
 424                           -DDISABLE_ECS_ACTIONS \
 425                           -DDISABLE_TOP_N_BINDINGS \
 426                           -DDISABLE_OCSP_STAPLING \
 427                           -DDISABLE_HASHED_CREDENTIALS \
 428                           -DDISABLE_FALSE_SHARING_PADDING \
 429                           -DDISABLE_NPN'
 430     unittests = ' --enable-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
 431     sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
 432     cflags = '-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int'
 433     cxxflags = cflags + ' -Wp,-D_GLIBCXX_ASSERTIONS ' + additional_flags
 434     res = c.run('''CFLAGS="%s" \
 435                    CXXFLAGS="%s" \
 436                    AR=llvm-ar-12 \
 437                    RANLIB=llvm-ranlib-12 \
 438                    ./configure \
 439                      CC='clang-12' \
 440                      CXX='clang++-12' \
 441                      --enable-option-checking=fatal \
 442                      --enable-fortify-source=auto \
 443                      --enable-auto-var-init=pattern \
 444                      --enable-lto=thin \
 445                      --prefix=/opt/dnsdist %s %s %s''' % (cflags, cxxflags, features_set, sanitizers, unittests), warn=True)
 446     if res.exited != 0:
 447         c.run('cat config.log')
 448         raise UnexpectedExit(res)
 449
 450 @task
 451 def ci_auth_make(c):
 452     c.run('make -j8 -k V=1')
 453
 454 @task
 455 def ci_rec_make(c):
 456     c.run('make -j8 -k V=1')
 457
 458 @task
 459 def ci_dnsdist_make(c):
 460     c.run('make -j4 -k V=1')
 461
 462 @task
 463 def ci_auth_install_remotebackend_test_deps(c):
 464     with c.cd('modules/remotebackend'):
 465       # c.run('bundle config set path vendor/bundle')
 466       c.run('sudo ruby -S bundle install')
 467     c.sudo('apt-get install -qq -y socat')
 468
 469 @task
 470 def ci_auth_run_unit_tests(c):
 471     res = c.run('make check', warn=True)
 472     if res.exited != 0:
 473       c.run('cat pdns/test-suite.log', warn=True)
 474       c.run('cat modules/remotebackend/test-suite.log', warn=True)
 475       raise UnexpectedExit(res)
 476
 477 @task
 478 def ci_rec_run_unit_tests(c):
 479     res = c.run('make check', warn=True)
 480     if res.exited != 0:
 481       c.run('cat test-suite.log')
 482       raise UnexpectedExit(res)
 483
 484 @task
 485 def ci_dnsdist_run_unit_tests(c):
 486     res = c.run('make check', warn=True)
 487     if res.exited != 0:
 488       c.run('cat test-suite.log')
 489       raise UnexpectedExit(res)
 490
 491 @task
 492 def ci_make_install(c):
 493     res = c.run('make install') # FIXME: this builds auth docs - again
 494
 495 @task
 496 def add_auth_repo(c):
 497     dist = 'ubuntu' # FIXME take these from the caller?
 498     release = 'focal'
 499     version = '44'
 500
 501     c.sudo('apt-get install -qq -y curl gnupg2')
 502     if version == 'master':
 503         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc')
 504     else:
 505         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc')
 506     c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist} {release}-auth-{version} main' | sudo tee /etc/apt/sources.list.d/pdns.list")
 507     c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns")
 508     c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns")
 509     c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns")
 510     c.sudo('apt-get update')
 511
 512 @task
 513 def test_api(c, product, backend=''):
 514     if product == 'recursor':
 515         with c.cd('regression-tests.api'):
 516             c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}')
 517     elif product == 'auth':
 518         with c.cd('regression-tests.api'):
 519             c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests authoritative {backend}')
 520     else:
 521         raise Failure('unknown product')
 522
 523 backend_regress_tests = dict(
 524     bind = [
 525         'bind-both',
 526         'bind-dnssec-both',
 527         'bind-dnssec-nsec3-both',
 528         'bind-dnssec-nsec3-optout-both',
 529         'bind-dnssec-nsec3-narrow',
 530         # FIXME  'bind-dnssec-pkcs11'
 531     ],
 532     geoip = [
 533         'geoip',
 534         'geoip-nsec3-narrow'
 535     ],
 536     lua2 = ['lua2', 'lua2-dnssec'],
 537     tinydns = ['tinydns'],
 538     remote = [
 539         'remotebackend-pipe',
 540         'remotebackend-unix',
 541         'remotebackend-http',
 542         'remotebackend-zeromq',
 543         'remotebackend-pipe-dnssec',
 544         'remotebackend-unix-dnssec',
 545         'remotebackend-http-dnssec',
 546         'remotebackend-zeromq-dnssec'
 547     ],
 548     lmdb = [
 549         'lmdb-nodnssec-both',
 550         'lmdb-both',
 551         'lmdb-nsec3-both',
 552         'lmdb-nsec3-optout-both',
 553         'lmdb-nsec3-narrow'
 554     ],
 555     gmysql = [
 556         'gmysql',
 557         'gmysql-nodnssec-both',
 558         'gmysql-nsec3-both',
 559         'gmysql-nsec3-optout-both',
 560         'gmysql-nsec3-narrow',
 561         'gmysql_sp-both'
 562     ],
 563     gpgsql = [
 564         'gpgsql',
 565         'gpgsql-nodnssec-both',
 566         'gpgsql-nsec3-both',
 567         'gpgsql-nsec3-optout-both',
 568         'gpgsql-nsec3-narrow',
 569         'gpgsql_sp-both'
 570     ],
 571     gsqlite3 = [
 572         'gsqlite3',
 573         'gsqlite3-nodnssec-both',
 574         'gsqlite3-nsec3-both',
 575         'gsqlite3-nsec3-optout-both',
 576         'gsqlite3-nsec3-narrow'
 577     ],
 578     godbc_sqlite3 = ['godbc_sqlite3-nodnssec'],
 579     godbc_mssql = [
 580         'godbc_mssql',
 581         'godbc_mssql-nodnssec',
 582         'godbc_mssql-nsec3',
 583         'godbc_mssql-nsec3-optout',
 584         'godbc_mssql-nsec3-narrow'
 585     ],
 586     ldap = [
 587         'ldap-tree',
 588         'ldap-simple',
 589         'ldap-strict'
 590     ],
 591     geoip_mmdb = ['geoip'],
 592 )
 593
 594 godbc_mssql_credentials = {"username": "sa", "password": "SAsa12%%"}
 595
 596 godbc_config = '''
 597 [pdns-mssql-docker]
 598 Driver=FreeTDS
 599 Trace=No
 600 Server=127.0.0.1
 601 Port=1433
 602 Database=pdns
 603 TDS_Version=7.1
 604
 605 [pdns-mssql-docker-nodb]
 606 Driver=FreeTDS
 607 Trace=No
 608 Server=127.0.0.1
 609 Port=1433
 610 TDS_Version=7.1
 611
 612 [pdns-sqlite3-1]
 613 Driver = SQLite3
 614 Database = pdns.sqlite3
 615
 616 [pdns-sqlite3-2]
 617 Driver = SQLite3
 618 Database = pdns.sqlite32
 619 '''
 620
 621 def setup_godbc_mssql(c):
 622     with open(os.path.expanduser("~/.odbc.ini"), "a") as f:
 623         f.write(godbc_config)
 624     c.sudo('sh -c \'echo "Threading=1" | cat /usr/share/tdsodbc/odbcinst.ini - | tee -a /etc/odbcinst.ini\'')
 625     c.sudo('sed -i "s/libtdsodbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libtdsodbc.so/g" /etc/odbcinst.ini')
 626     c.run(f'echo "create database pdns" | isql -v pdns-mssql-docker-nodb {godbc_mssql_credentials["username"]} {godbc_mssql_credentials["password"]}')
 627     # FIXME: Skip 8bit-txt-unescaped test
 628     c.run('touch ${PWD}/regression-tests/tests/8bit-txt-unescaped/skip')
 629
 630 def setup_godbc_sqlite3(c):
 631     with open(os.path.expanduser("~/.odbc.ini"), "a") as f:
 632         f.write(godbc_config)
 633     c.sudo('sed -i "s/libsqlite3odbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libsqlite3odbc.so/g" /etc/odbcinst.ini')
 634
 635 def setup_ldap_client(c):
 636     c.sudo('DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ldap-utils')
 637     c.sudo('sh -c \'echo "127.0.0.1 ldapserver" | tee -a /etc/hosts\'')
 638
 639 @task
 640 def test_auth_backend(c, backend):
 641     pdns_auth_env_vars = 'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432"'
 642
 643     if backend == 'remote':
 644         ci_auth_install_remotebackend_test_deps(c)
 645
 646     if backend == 'authpy':
 647         with c.cd('regression-tests.auth-py'):
 648             c.run(f'{pdns_auth_env_vars} WITHKERBEROS=YES ./runtests')
 649         return
 650
 651     if backend == 'godbc_sqlite3':
 652         setup_godbc_sqlite3(c)
 653         with c.cd('regression-tests'):
 654             for variant in backend_regress_tests[backend]:
 655                 c.run(f'{pdns_auth_env_vars} GODBC_SQLITE3_DSN=pdns-sqlite3-1 ./start-test-stop 5300 {variant}')
 656         return
 657
 658     if backend == 'godbc_mssql':
 659         setup_godbc_mssql(c)
 660         with c.cd('regression-tests'):
 661             for variant in backend_regress_tests[backend]:
 662                 c.run(f'{pdns_auth_env_vars} GODBC_MSSQL_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL_DSN=pdns-mssql-docker GODBC_MSSQL2_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL2_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL2_DSN=pdns-mssql-docker ./start-test-stop 5300 {variant}')
 663         return
 664
 665     if backend == 'ldap':
 666         setup_ldap_client(c)
 667
 668     if backend == 'geoip_mmdb':
 669         with c.cd('regression-tests'):
 670             for variant in backend_regress_tests[backend]:
 671                 c.run(f'{pdns_auth_env_vars} geoipdatabase=../modules/geoipbackend/regression-tests/GeoLiteCity.mmdb ./start-test-stop 5300 {variant}')
 672         return
 673
 674     with c.cd('regression-tests'):
 675         if backend == 'lua2':
 676             c.run('touch trustedkeys')  # avoid silly error during cleanup
 677         for variant in backend_regress_tests[backend]:
 678             c.run(f'{pdns_auth_env_vars} ./start-test-stop 5300 {variant}')
 679
 680     if backend == 'gsqlite3':
 681         with c.cd('regression-tests.nobackend'):
 682             c.run(f'{pdns_auth_env_vars} ./runtests')
 683         c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms')
 684         return
 685
 686 @task
 687 def test_ixfrdist(c):
 688     with c.cd('regression-tests.ixfrdist'):
 689         c.run('IXFRDISTBIN=/opt/pdns-auth/bin/ixfrdist ./runtests')
 690
 691 @task
 692 def test_dnsdist(c):
 693     c.run('chmod +x /opt/dnsdist/bin/*')
 694     c.run('ls -ald /var /var/agentx /var/agentx/master')
 695     c.run('ls -al /var/agentx/master')
 696     with c.cd('regression-tests.dnsdist'):
 697         c.run('DNSDISTBIN=/opt/dnsdist/bin/dnsdist ./runtests')
 698
 699 @task
 700 def test_regression_recursor(c):
 701     c.run('/opt/pdns-recursor/sbin/pdns_recursor --version')
 702     c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control SKIP_IPV6_TESTS=y ./build-scripts/test-recursor')
 703
 704 @task
 705 def test_bulk_recursor(c, threads, mthreads, shards):
 706     # We run an extremely small version of the bulk test, as GH does not seem to be able to handle the UDP load
 707     with c.cd('regression-tests'):
 708         c.run('curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip')
 709         c.run('unzip top-1m.csv.zip -d .')
 710         c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 711         c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control THRESHOLD=95 TRACE=no ./timestamp ./recursor-test 5300 100 {threads} {mthreads} {shards}')
 712
 713 @task
 714 def install_swagger_tools(c):
 715     c.run('npm install -g api-spec-converter')
 716
 717 @task
 718 def swagger_syntax_check(c):
 719     c.run('api-spec-converter docs/http-api/swagger/authoritative-api-swagger.yaml -f swagger_2 -t openapi_3 -s json -c')
 720
 721 @task
 722 def install_coverity_tools(c, project):
 723     token = os.getenv('COVERITY_TOKEN')
 724     c.run(f'curl -s https://scan.coverity.com/download/linux64 --data "token={token}&project={project}" | gunzip | sudo tar xvf /dev/stdin --strip-components=1 --no-same-owner -C /usr/local', hide=True)
 725
 726 @task
 727 def coverity_clang_configure(c):
 728     c.sudo('/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-12')
 729
 730 @task
 731 def coverity_make(c):
 732     c.run('/usr/local/bin/cov-build --dir cov-int make -j8 -k')
 733
 734 @task
 735 def coverity_tarball(c, tarball):
 736     c.run(f'tar caf {tarball} cov-int')
 737
 738 @task
 739 def coverity_upload(c, email, project, tarball):
 740     token = os.getenv('COVERITY_TOKEN')
 741     c.run(f'curl --form token={token} \
 742             --form email="{email}" \
 743             --form file=@{tarball} \
 744             --form version="$(./builder-support/gen-version)" \
 745             --form description="master build" \
 746             https://scan.coverity.com/builds?project={project}', hide=True)
 747
 748 # this is run always
 749 def setup():
 750     if '/usr/lib/ccache' not in os.environ['PATH']:
 751         os.environ['PATH']='/usr/lib/ccache:'+os.environ['PATH']
 752
 753 setup()