tasks.py

   1 from invoke import task
   2 from invoke.exceptions import Failure, UnexpectedExit
   3
   4 import os
   5 import sys
   6 import time
   7
   8 all_build_deps = [
   9     'ccache',
  10     'libboost-all-dev',
  11     'libluajit-5.1-dev',
  12     'libsodium-dev',
  13     'libssl-dev',
  14     'libsystemd-dev',
  15     'libtool',
  16     'make',
  17     'pkg-config',
  18     'python3-venv',
  19     'systemd',
  20 ]
  21 git_build_deps = [
  22     'autoconf',
  23     'automake',
  24     'bison',
  25     'bzip2',
  26     'curl',
  27     'flex',
  28     'git',
  29     'ragel'
  30 ]
  31 auth_build_deps = [    # FIXME: perhaps we should be stealing these from the debian (Ubuntu) control file
  32     'default-libmysqlclient-dev',
  33     'libcdb-dev',
  34     'libcurl4-openssl-dev',
  35     'libgeoip-dev',
  36     'libkrb5-dev',
  37     'libldap2-dev',
  38     'liblmdb-dev',
  39     'libmaxminddb-dev',
  40     'libp11-kit-dev',
  41     'libpq-dev',
  42     'libsqlite3-dev',
  43     'libyaml-cpp-dev',
  44     'libzmq3-dev',
  45     'ruby-bundler',
  46     'ruby-dev',
  47     'sqlite3',
  48     'unixodbc-dev',
  49     'cmake',
  50 ]
  51 rec_build_deps = [
  52     'libcap-dev',
  53     'libfstrm-dev',
  54     'libsnmp-dev',
  55 ]
  56 rec_bulk_deps = [
  57     'curl',
  58     'libboost-all-dev',
  59     'libcap2',
  60     'libfstrm0',
  61     'libluajit-5.1-2',
  62     'libsnmp35',
  63     'libsodium23',
  64     'libssl1.1',
  65     'libsystemd0',
  66     'moreutils',
  67     'pdns-tools',
  68     'unzip',
  69 ]
  70 dnsdist_build_deps = [
  71     'libcap-dev',
  72     'libcdb-dev',
  73     'libedit-dev',
  74     'libfstrm-dev',
  75     'libgnutls28-dev',
  76     'libh2o-evloop-dev',
  77     'liblmdb-dev',
  78     'libnghttp2-dev',
  79     'libre2-dev',
  80     'libsnmp-dev',
  81 ]
  82 auth_test_deps = [   # FIXME: we should be generating some of these from shlibdeps in build
  83     'authbind',
  84     'bc',
  85     'bind9utils',
  86     'curl',
  87     'default-jre-headless',
  88     'dnsutils',
  89     'docker-compose',
  90     'faketime',
  91     'gawk',
  92     'krb5-user',
  93     'ldnsutils',
  94     'libboost-serialization1.71.0',
  95     'libcdb1',
  96     'libcurl4',
  97     'libgeoip1',
  98     'libkrb5-3',
  99     'libldap-2.4-2',
 100     'liblmdb0',
 101     'libluajit-5.1-2',
 102     'libmaxminddb0',
 103     'libnet-dns-perl',
 104     'libp11-kit0',
 105     'libpq5',
 106     'libsodium23',
 107     'libsqlite3-dev',
 108     'libssl1.1',
 109     'libsystemd0',
 110     'libyaml-cpp0.6',
 111     'libzmq3-dev',
 112     'lmdb-utils',
 113     'prometheus',
 114     'ruby-bundler',
 115     'ruby-dev',
 116     'socat',
 117     'softhsm2',
 118     'unbound-host',
 119     'unixodbc',
 120     'wget',
 121 ]
 122 doc_deps = [
 123     'autoconf',
 124     'automake',
 125     'bison',
 126     'curl',
 127     'flex',
 128     'g++',
 129     'git',
 130     'latexmk',
 131     'libboost-all-dev',
 132     'libedit-dev',
 133     'libluajit-5.1-dev',
 134     'libssl-dev',
 135     'make',
 136     'pkg-config',
 137     'python3-venv',
 138     'ragel',
 139     'rsync',
 140 ]
 141 doc_deps_pdf = [
 142     'texlive-binaries',
 143     'texlive-formats-extra',
 144     'texlive-latex-extra',
 145 ]
 146
 147 @task
 148 def apt_fresh(c):
 149     c.sudo('sed -i \'s/azure\.//\' /etc/apt/sources.list')
 150     c.sudo('apt-get update')
 151     c.sudo('apt-get -qq -y --allow-downgrades dist-upgrade')
 152
 153 @task
 154 def install_clang(c):
 155     """
 156     install clang-12 and llvm-12
 157     """
 158     c.sudo('apt-get -qq -y --no-install-recommends install clang-12 llvm-12')
 159
 160 @task
 161 def install_clang_runtime(c):
 162     # this gives us the symbolizer, for symbols in asan/ubsan traces
 163     c.sudo('apt-get -qq -y --no-install-recommends install clang-12')
 164
 165 def install_libdecaf(c, product):
 166     c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf')
 167     with c.cd('/tmp/libdecaf'):
 168         c.run('git checkout 41f349')
 169         c.run('CC=clang-12 CXX=clang-12 '
 170               'cmake -B build '
 171               '-DCMAKE_INSTALL_PREFIX=/usr/local '
 172               '-DCMAKE_INSTALL_LIBDIR=lib '
 173               '-DENABLE_STATIC=OFF '
 174               '-DENABLE_TESTS=OFF '
 175               '-DCMAKE_C_FLAGS="-Wno-sizeof-array-div -Wno-array-parameter" .')
 176         c.run('make -C build')
 177         c.run('sudo make -C build install')
 178     c.sudo(f'mkdir -p /opt/{product}/libdecaf')
 179     c.sudo(f'cp /usr/local/lib/libdecaf.so* /opt/{product}/libdecaf/.')
 180
 181 @task
 182 def install_doc_deps(c):
 183     c.sudo('apt-get install -qq -y ' + ' '.join(doc_deps))
 184
 185 @task
 186 def install_doc_deps_pdf(c):
 187     c.sudo('apt-get install -qq -y ' + ' '.join(doc_deps_pdf))
 188
 189 @task
 190 def install_auth_build_deps(c):
 191     c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps))
 192     install_libdecaf(c, 'pdns-auth')
 193
 194 def setup_authbind(c):
 195     c.sudo('touch /etc/authbind/byport/53')
 196     c.sudo('chmod 755 /etc/authbind/byport/53')
 197
 198 auth_backend_test_deps = dict(
 199     gsqlite3=['sqlite3'],
 200     gmysql=['default-libmysqlclient-dev'],
 201     gpgsql=['libpq-dev'],
 202     lmdb=[],
 203     remote=[],
 204     bind=[],
 205     geoip=[],
 206     lua2=[],
 207     tinydns=[],
 208     authpy=[],
 209     godbc_sqlite3=['libsqliteodbc'],
 210     godbc_mssql=['freetds-bin','tdsodbc'],
 211     ldap=[],
 212     geoip_mmdb=[]
 213 )
 214
 215 @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend'])
 216 def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more than apt-get
 217     extra=[]
 218     for b in backend:
 219         extra.extend(auth_backend_test_deps[b])
 220     c.sudo('apt-get -y -qq install ' + ' '.join(extra+auth_test_deps))
 221
 222     c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*')
 223     # c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then
 224     #               wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz
 225     #               tar xfz jdnssec-tools-0.14.tar.gz -C $HOME
 226     #               rm jdnssec-tools-0.14.tar.gz
 227     #          fi
 228     #          echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''')  # FIXME: why did this fail with no error?
 229     c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing regression-tests.nobackend/rectify-axfr/allow-missing') # FIXME: can this go?
 230     # FIXME we may want to start a background recursor here to make ALIAS tests more robust
 231     setup_authbind(c)
 232
 233     # Copy libdecaf out
 234     c.sudo('mkdir -p /usr/local/lib')
 235     c.sudo('cp /opt/pdns-auth/libdecaf/libdecaf.so* /usr/local/lib/.')
 236
 237 @task
 238 def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get
 239     c.sudo('apt-get --no-install-recommends -qq -y install ' + ' '.join(rec_bulk_deps))
 240     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 241
 242 @task
 243 def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get
 244     c.sudo('apt-get --no-install-recommends install -qq -y ' + ' '.join(rec_bulk_deps) + ' \
 245               pdns-server pdns-backend-bind daemontools \
 246               jq libfaketime lua-posix lua-socket bc authbind \
 247               python3-venv python3-dev default-libmysqlclient-dev libpq-dev \
 248               protobuf-compiler snmpd prometheus')
 249
 250     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 251
 252     setup_authbind(c)
 253
 254     c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 255     c.sudo('systemctl restart snmpd')
 256     time.sleep(5)
 257     c.sudo('chmod 755 /var/agentx')
 258
 259 @task
 260 def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt-get
 261     c.sudo('apt-get install -qq -y \
 262               libluajit-5.1-2 \
 263               libboost-all-dev \
 264               libcap2 \
 265               libcdb1 \
 266               libcurl4-openssl-dev \
 267               libfstrm0 \
 268               libgnutls30 \
 269               libh2o-evloop0.13 \
 270               liblmdb0 \
 271               libnghttp2-14 \
 272               libre2-5 \
 273               libssl-dev \
 274               libsystemd0 \
 275               libsodium23 \
 276               lua-socket \
 277               patch \
 278               protobuf-compiler \
 279               python3-venv snmpd prometheus')
 280     c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 281     c.sudo('systemctl restart snmpd')
 282     time.sleep(5)
 283     c.sudo('chmod 755 /var/agentx')
 284
 285 @task
 286 def install_rec_build_deps(c):
 287     c.sudo('apt-get install -qq -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + rec_build_deps))
 288
 289 @task
 290 def install_dnsdist_build_deps(c):
 291     c.sudo('apt-get install -qq -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps))
 292
 293 @task
 294 def ci_autoconf(c):
 295     c.run('BUILDER_VERSION=0.0.0-git1 autoreconf -vfi')
 296
 297 @task
 298 def ci_docs_build(c):
 299     c.run('make -f Makefile.sphinx -C docs html')
 300
 301 @task
 302 def ci_docs_build_pdf(c):
 303     c.run('make -f Makefile.sphinx -C docs latexpdf')
 304
 305 @task
 306 def ci_docs_upload_master(c, docs_host, pdf, username, product, directory=""):
 307     rsync_cmd = " ".join([
 308         "rsync",
 309         "--checksum",
 310         "--recursive",
 311         "--verbose",
 312         "--no-p",
 313         "--chmod=g=rwX",
 314         "--exclude '*~'",
 315     ])
 316     c.run(f"{rsync_cmd} --delete ./docs/_build/{product}-html-docs/ {username}@{docs_host}:{directory}")
 317     c.run(f"{rsync_cmd} ./docs/_build/{product}-html-docs.tar.bz2 {username}@{docs_host}:{directory}/html-docs.tar.bz2")
 318     c.run(f"{rsync_cmd} ./docs/_build/latex/{pdf} {username}@{docs_host}:{directory}")
 319
 320 @task
 321 def ci_docs_add_ssh(c, ssh_key, host_key):
 322     c.run('mkdir -m 700 -p ~/.ssh')
 323     c.run(f'echo "{ssh_key}" > ~/.ssh/id_ed25519')
 324     c.run('chmod 600 ~/.ssh/id_ed25519')
 325     c.run(f'echo "{host_key}" > ~/.ssh/known_hosts')
 326
 327
 328 def get_sanitizers():
 329     sanitizers = os.getenv('SANITIZERS')
 330     if sanitizers != '':
 331         sanitizers = sanitizers.split('+')
 332         sanitizers = ['--enable-' + sanitizer for sanitizer in sanitizers]
 333         sanitizers = ' '.join(sanitizers)
 334     return sanitizers
 335
 336
 337 def get_cflags():
 338     return " ".join([
 339         "-O1",
 340         "-Werror=vla",
 341         "-Werror=shadow",
 342         "-Wformat=2",
 343         "-Werror=format-security",
 344         "-Werror=string-plus-int",
 345     ])
 346
 347
 348 def get_cxxflags():
 349     return " ".join([
 350         get_cflags(),
 351         "-Wp,-D_GLIBCXX_ASSERTIONS",
 352     ])
 353
 354
 355 def get_base_configure_cmd():
 356     return " ".join([
 357         f'CFLAGS="{get_cflags()}"',
 358         f'CXXFLAGS="{get_cxxflags()}"',
 359         './configure',
 360         "CC='clang-12'",
 361         "CXX='clang++-12'",
 362         "--enable-option-checking=fatal",
 363         "--enable-systemd",
 364         "--with-libsodium",
 365         "--enable-fortify-source=auto",
 366         "--enable-auto-var-init=pattern",
 367     ])
 368
 369
 370 @task
 371 def ci_auth_configure(c):
 372     sanitizers = get_sanitizers()
 373
 374     unittests = os.getenv('UNIT_TESTS')
 375     if unittests == 'yes':
 376         unittests = '--enable-unit-tests --enable-backend-unit-tests'
 377     else:
 378         unittests = ''
 379
 380     fuzz_targets = os.getenv('FUZZING_TARGETS')
 381     fuzz_targets = '--enable-fuzz-targets' if fuzz_targets == 'yes' else ''
 382
 383     modules = " ".join([
 384         "bind",
 385         "geoip",
 386         "gmysql",
 387         "godbc",
 388         "gpgsql",
 389         "gsqlite3",
 390         "ldap",
 391         "lmdb",
 392         "lua2",
 393         "pipe",
 394         "remote",
 395         "tinydns",
 396     ])
 397     configure_cmd = " ".join([
 398         get_base_configure_cmd(),
 399         "LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib'",
 400         f"--with-modules='{modules}'",
 401         "--enable-tools",
 402         "--enable-experimental-pkcs11",
 403         "--enable-experimental-gss-tsig",
 404         "--enable-remotebackend-zeromq",
 405         "--with-lmdb=/usr",
 406         "--with-libdecaf",
 407         "--prefix=/opt/pdns-auth",
 408         "--enable-ixfrdist",
 409         sanitizers,
 410         unittests,
 411         fuzz_targets,
 412     ])
 413     res = c.run(configure_cmd, warn=True)
 414     if res.exited != 0:
 415         c.run('cat config.log')
 416         raise UnexpectedExit(res)
 417
 418
 419 @task
 420 def ci_rec_configure(c):
 421     sanitizers = get_sanitizers()
 422
 423     unittests = os.getenv('UNIT_TESTS')
 424     unittests = '--enable-unit-tests' if unittests == 'yes' else ''
 425
 426     configure_cmd = " ".join([
 427         get_base_configure_cmd(),
 428         "--enable-nod",
 429         "--prefix=/opt/pdns-recursor",
 430         "--with-lua=luajit",
 431         "--with-libcap",
 432         "--with-net-snmp",
 433         "--enable-dns-over-tls",
 434         sanitizers,
 435         unittests,
 436     ])
 437     res = c.run(configure_cmd, warn=True)
 438     if res.exited != 0:
 439         c.run('cat config.log')
 440         raise UnexpectedExit(res)
 441
 442
 443 @task
 444 def ci_dnsdist_configure(c, features):
 445     additional_flags = ''
 446     if features == 'full':
 447       features_set = '--enable-dnstap \
 448                       --enable-dnscrypt \
 449                       --enable-dns-over-tls \
 450                       --enable-dns-over-https \
 451                       --enable-systemd \
 452                       --prefix=/opt/dnsdist \
 453                       --with-gnutls \
 454                       --with-libsodium \
 455                       --with-lua=luajit \
 456                       --with-libcap \
 457                       --with-nghttp2 \
 458                       --with-re2 '
 459     else:
 460       features_set = '--disable-dnstap \
 461                       --disable-dnscrypt \
 462                       --disable-ipcipher \
 463                       --disable-systemd \
 464                       --without-cdb \
 465                       --without-ebpf \
 466                       --without-gnutls \
 467                       --without-libedit \
 468                       --without-libsodium \
 469                       --without-lmdb \
 470                       --without-net-snmp \
 471                       --without-nghttp2 \
 472                       --without-re2 '
 473       additional_flags = '-DDISABLE_COMPLETION \
 474                           -DDISABLE_DELAY_PIPE \
 475                           -DDISABLE_DYNBLOCKS \
 476                           -DDISABLE_PROMETHEUS \
 477                           -DDISABLE_PROTOBUF \
 478                           -DDISABLE_BUILTIN_HTML \
 479                           -DDISABLE_CARBON \
 480                           -DDISABLE_SECPOLL \
 481                           -DDISABLE_DEPRECATED_DYNBLOCK \
 482                           -DDISABLE_LUA_WEB_HANDLERS \
 483                           -DDISABLE_NON_FFI_DQ_BINDINGS \
 484                           -DDISABLE_POLICIES_BINDINGS \
 485                           -DDISABLE_PACKETCACHE_BINDINGS \
 486                           -DDISABLE_DOWNSTREAM_BINDINGS \
 487                           -DDISABLE_COMBO_ADDR_BINDINGS \
 488                           -DDISABLE_CLIENT_STATE_BINDINGS \
 489                           -DDISABLE_QPS_LIMITER_BINDINGS \
 490                           -DDISABLE_SUFFIX_MATCH_BINDINGS \
 491                           -DDISABLE_NETMASK_BINDINGS \
 492                           -DDISABLE_DNSNAME_BINDINGS \
 493                           -DDISABLE_DNSHEADER_BINDINGS \
 494                           -DDISABLE_RECVMMSG \
 495                           -DDISABLE_WEB_CACHE_MANAGEMENT \
 496                           -DDISABLE_WEB_CONFIG \
 497                           -DDISABLE_RULES_ALTERING_QUERIES \
 498                           -DDISABLE_ECS_ACTIONS \
 499                           -DDISABLE_TOP_N_BINDINGS \
 500                           -DDISABLE_OCSP_STAPLING \
 501                           -DDISABLE_HASHED_CREDENTIALS \
 502                           -DDISABLE_FALSE_SHARING_PADDING \
 503                           -DDISABLE_NPN'
 504     unittests = ' --enable-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
 505     sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
 506     cflags = '-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int'
 507     cxxflags = cflags + ' -Wp,-D_GLIBCXX_ASSERTIONS ' + additional_flags
 508     res = c.run('''CFLAGS="%s" \
 509                    CXXFLAGS="%s" \
 510                    AR=llvm-ar-12 \
 511                    RANLIB=llvm-ranlib-12 \
 512                    ./configure \
 513                      CC='clang-12' \
 514                      CXX='clang++-12' \
 515                      --enable-option-checking=fatal \
 516                      --enable-fortify-source=auto \
 517                      --enable-auto-var-init=pattern \
 518                      --enable-lto=thin \
 519                      --prefix=/opt/dnsdist %s %s %s''' % (cflags, cxxflags, features_set, sanitizers, unittests), warn=True)
 520     if res.exited != 0:
 521         c.run('cat config.log')
 522         raise UnexpectedExit(res)
 523
 524 @task
 525 def ci_auth_make(c):
 526     c.run('make -j8 -k V=1')
 527
 528 @task
 529 def ci_rec_make(c):
 530     c.run('make -j8 -k V=1')
 531
 532 @task
 533 def ci_dnsdist_make(c):
 534     c.run('make -j4 -k V=1')
 535
 536 @task
 537 def ci_auth_install_remotebackend_test_deps(c):
 538     with c.cd('modules/remotebackend'):
 539       # c.run('bundle config set path vendor/bundle')
 540       c.run('sudo ruby -S bundle install')
 541     c.sudo('apt-get install -qq -y socat')
 542
 543 @task
 544 def ci_auth_run_unit_tests(c):
 545     res = c.run('make check', warn=True)
 546     if res.exited != 0:
 547       c.run('cat pdns/test-suite.log', warn=True)
 548       c.run('cat modules/remotebackend/test-suite.log', warn=True)
 549       raise UnexpectedExit(res)
 550
 551 @task
 552 def ci_rec_run_unit_tests(c):
 553     res = c.run('make check', warn=True)
 554     if res.exited != 0:
 555       c.run('cat test-suite.log')
 556       raise UnexpectedExit(res)
 557
 558 @task
 559 def ci_dnsdist_run_unit_tests(c):
 560     res = c.run('make check', warn=True)
 561     if res.exited != 0:
 562       c.run('cat test-suite.log')
 563       raise UnexpectedExit(res)
 564
 565 @task
 566 def ci_make_install(c):
 567     res = c.run('make install') # FIXME: this builds auth docs - again
 568
 569 @task
 570 def add_auth_repo(c):
 571     dist = 'ubuntu' # FIXME take these from the caller?
 572     release = 'focal'
 573     version = '44'
 574
 575     c.sudo('apt-get install -qq -y curl gnupg2')
 576     if version == 'master':
 577         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc')
 578     else:
 579         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc')
 580     c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist} {release}-auth-{version} main' | sudo tee /etc/apt/sources.list.d/pdns.list")
 581     c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns")
 582     c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns")
 583     c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns")
 584     c.sudo('apt-get update')
 585
 586 @task
 587 def test_api(c, product, backend=''):
 588     if product == 'recursor':
 589         with c.cd('regression-tests.api'):
 590             c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}')
 591     elif product == 'auth':
 592         with c.cd('regression-tests.api'):
 593             c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests authoritative {backend}')
 594     else:
 595         raise Failure('unknown product')
 596
 597 backend_regress_tests = dict(
 598     bind = [
 599         'bind-both',
 600         'bind-dnssec-both',
 601         'bind-dnssec-nsec3-both',
 602         'bind-dnssec-nsec3-optout-both',
 603         'bind-dnssec-nsec3-narrow',
 604         # FIXME  'bind-dnssec-pkcs11'
 605     ],
 606     geoip = [
 607         'geoip',
 608         'geoip-nsec3-narrow'
 609     ],
 610     lua2 = ['lua2', 'lua2-dnssec'],
 611     tinydns = ['tinydns'],
 612     remote = [
 613         'remotebackend-pipe',
 614         'remotebackend-unix',
 615         'remotebackend-http',
 616         'remotebackend-zeromq',
 617         'remotebackend-pipe-dnssec',
 618         'remotebackend-unix-dnssec',
 619         'remotebackend-http-dnssec',
 620         'remotebackend-zeromq-dnssec'
 621     ],
 622     lmdb = [
 623         'lmdb-nodnssec-both',
 624         'lmdb-both',
 625         'lmdb-nsec3-both',
 626         'lmdb-nsec3-optout-both',
 627         'lmdb-nsec3-narrow'
 628     ],
 629     gmysql = [
 630         'gmysql',
 631         'gmysql-nodnssec-both',
 632         'gmysql-nsec3-both',
 633         'gmysql-nsec3-optout-both',
 634         'gmysql-nsec3-narrow',
 635         'gmysql_sp-both'
 636     ],
 637     gpgsql = [
 638         'gpgsql',
 639         'gpgsql-nodnssec-both',
 640         'gpgsql-nsec3-both',
 641         'gpgsql-nsec3-optout-both',
 642         'gpgsql-nsec3-narrow',
 643         'gpgsql_sp-both'
 644     ],
 645     gsqlite3 = [
 646         'gsqlite3',
 647         'gsqlite3-nodnssec-both',
 648         'gsqlite3-nsec3-both',
 649         'gsqlite3-nsec3-optout-both',
 650         'gsqlite3-nsec3-narrow'
 651     ],
 652     godbc_sqlite3 = ['godbc_sqlite3-nodnssec'],
 653     godbc_mssql = [
 654         'godbc_mssql',
 655         'godbc_mssql-nodnssec',
 656         'godbc_mssql-nsec3',
 657         'godbc_mssql-nsec3-optout',
 658         'godbc_mssql-nsec3-narrow'
 659     ],
 660     ldap = [
 661         'ldap-tree',
 662         'ldap-simple',
 663         'ldap-strict'
 664     ],
 665     geoip_mmdb = ['geoip'],
 666 )
 667
 668 godbc_mssql_credentials = {"username": "sa", "password": "SAsa12%%"}
 669
 670 godbc_config = '''
 671 [pdns-mssql-docker]
 672 Driver=FreeTDS
 673 Trace=No
 674 Server=127.0.0.1
 675 Port=1433
 676 Database=pdns
 677 TDS_Version=7.1
 678
 679 [pdns-mssql-docker-nodb]
 680 Driver=FreeTDS
 681 Trace=No
 682 Server=127.0.0.1
 683 Port=1433
 684 TDS_Version=7.1
 685
 686 [pdns-sqlite3-1]
 687 Driver = SQLite3
 688 Database = pdns.sqlite3
 689
 690 [pdns-sqlite3-2]
 691 Driver = SQLite3
 692 Database = pdns.sqlite32
 693 '''
 694
 695 def setup_godbc_mssql(c):
 696     with open(os.path.expanduser("~/.odbc.ini"), "a") as f:
 697         f.write(godbc_config)
 698     c.sudo('sh -c \'echo "Threading=1" | cat /usr/share/tdsodbc/odbcinst.ini - | tee -a /etc/odbcinst.ini\'')
 699     c.sudo('sed -i "s/libtdsodbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libtdsodbc.so/g" /etc/odbcinst.ini')
 700     c.run(f'echo "create database pdns" | isql -v pdns-mssql-docker-nodb {godbc_mssql_credentials["username"]} {godbc_mssql_credentials["password"]}')
 701     # FIXME: Skip 8bit-txt-unescaped test
 702     c.run('touch ${PWD}/regression-tests/tests/8bit-txt-unescaped/skip')
 703
 704 def setup_godbc_sqlite3(c):
 705     with open(os.path.expanduser("~/.odbc.ini"), "a") as f:
 706         f.write(godbc_config)
 707     c.sudo('sed -i "s/libsqlite3odbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libsqlite3odbc.so/g" /etc/odbcinst.ini')
 708
 709 def setup_ldap_client(c):
 710     c.sudo('DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ldap-utils')
 711     c.sudo('sh -c \'echo "127.0.0.1 ldapserver" | tee -a /etc/hosts\'')
 712
 713 @task
 714 def test_auth_backend(c, backend):
 715     pdns_auth_env_vars = 'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432"'
 716
 717     if backend == 'remote':
 718         ci_auth_install_remotebackend_test_deps(c)
 719
 720     if backend == 'authpy':
 721         with c.cd('regression-tests.auth-py'):
 722             c.run(f'{pdns_auth_env_vars} WITHKERBEROS=YES ./runtests')
 723         return
 724
 725     if backend == 'godbc_sqlite3':
 726         setup_godbc_sqlite3(c)
 727         with c.cd('regression-tests'):
 728             for variant in backend_regress_tests[backend]:
 729                 c.run(f'{pdns_auth_env_vars} GODBC_SQLITE3_DSN=pdns-sqlite3-1 ./start-test-stop 5300 {variant}')
 730         return
 731
 732     if backend == 'godbc_mssql':
 733         setup_godbc_mssql(c)
 734         with c.cd('regression-tests'):
 735             for variant in backend_regress_tests[backend]:
 736                 c.run(f'{pdns_auth_env_vars} GODBC_MSSQL_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL_DSN=pdns-mssql-docker GODBC_MSSQL2_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL2_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL2_DSN=pdns-mssql-docker ./start-test-stop 5300 {variant}')
 737         return
 738
 739     if backend == 'ldap':
 740         setup_ldap_client(c)
 741
 742     if backend == 'geoip_mmdb':
 743         with c.cd('regression-tests'):
 744             for variant in backend_regress_tests[backend]:
 745                 c.run(f'{pdns_auth_env_vars} geoipdatabase=../modules/geoipbackend/regression-tests/GeoLiteCity.mmdb ./start-test-stop 5300 {variant}')
 746         return
 747
 748     with c.cd('regression-tests'):
 749         if backend == 'lua2':
 750             c.run('touch trustedkeys')  # avoid silly error during cleanup
 751         for variant in backend_regress_tests[backend]:
 752             c.run(f'{pdns_auth_env_vars} ./start-test-stop 5300 {variant}')
 753
 754     if backend == 'gsqlite3':
 755         with c.cd('regression-tests.nobackend'):
 756             c.run(f'{pdns_auth_env_vars} ./runtests')
 757         c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms')
 758         return
 759
 760 @task
 761 def test_ixfrdist(c):
 762     with c.cd('regression-tests.ixfrdist'):
 763         c.run('IXFRDISTBIN=/opt/pdns-auth/bin/ixfrdist ./runtests')
 764
 765 @task
 766 def test_dnsdist(c):
 767     c.run('chmod +x /opt/dnsdist/bin/*')
 768     c.run('ls -ald /var /var/agentx /var/agentx/master')
 769     c.run('ls -al /var/agentx/master')
 770     with c.cd('regression-tests.dnsdist'):
 771         c.run('DNSDISTBIN=/opt/dnsdist/bin/dnsdist ./runtests')
 772
 773 @task
 774 def test_regression_recursor(c):
 775     c.run('/opt/pdns-recursor/sbin/pdns_recursor --version')
 776     c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control SKIP_IPV6_TESTS=y ./build-scripts/test-recursor')
 777
 778 @task
 779 def test_bulk_recursor(c, threads, mthreads, shards):
 780     # We run an extremely small version of the bulk test, as GH does not seem to be able to handle the UDP load
 781     with c.cd('regression-tests'):
 782         c.run('curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip')
 783         c.run('unzip top-1m.csv.zip -d .')
 784         c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 785         c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control THRESHOLD=95 TRACE=no ./timestamp ./recursor-test 5300 100 {threads} {mthreads} {shards}')
 786
 787 @task
 788 def install_swagger_tools(c):
 789     c.run('npm install -g api-spec-converter')
 790
 791 @task
 792 def swagger_syntax_check(c):
 793     c.run('api-spec-converter docs/http-api/swagger/authoritative-api-swagger.yaml -f swagger_2 -t openapi_3 -s json -c')
 794
 795 @task
 796 def install_coverity_tools(c, project):
 797     token = os.getenv('COVERITY_TOKEN')
 798     c.run(f'curl -s https://scan.coverity.com/download/linux64 --data "token={token}&project={project}" | gunzip | sudo tar xvf /dev/stdin --strip-components=1 --no-same-owner -C /usr/local', hide=True)
 799
 800 @task
 801 def coverity_clang_configure(c):
 802     c.sudo('/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-12')
 803
 804 @task
 805 def coverity_make(c):
 806     c.run('/usr/local/bin/cov-build --dir cov-int make -j8 -k')
 807
 808 @task
 809 def coverity_tarball(c, tarball):
 810     c.run(f'tar caf {tarball} cov-int')
 811
 812 @task
 813 def coverity_upload(c, email, project, tarball):
 814     token = os.getenv('COVERITY_TOKEN')
 815     c.run(f'curl --form token={token} \
 816             --form email="{email}" \
 817             --form file=@{tarball} \
 818             --form version="$(./builder-support/gen-version)" \
 819             --form description="master build" \
 820             https://scan.coverity.com/builds?project={project}', hide=True)
 821
 822 # this is run always
 823 def setup():
 824     if '/usr/lib/ccache' not in os.environ['PATH']:
 825         os.environ['PATH']='/usr/lib/ccache:'+os.environ['PATH']
 826
 827 setup()