tasks.py

   1 from invoke import task
   2 from invoke.exceptions import Failure, UnexpectedExit
   3
   4 import os
   5 import sys
   6 import time
   7
   8 auth_backend_ip_addr = os.getenv('AUTH_BACKEND_IP_ADDR', '127.0.0.1')
   9
  10 clang_version = os.getenv('CLANG_VERSION', '13')
  11 quiche_version = '0.20.0'
  12 quiche_hash = '7125bc82ddcf38fbfbc69882ccb2723bfb4d5bfeb42718b8291d26ec06042e38'
  13
  14 all_build_deps = [
  15     'ccache',
  16     'libboost-all-dev',
  17     'libluajit-5.1-dev',
  18     'libsodium-dev',
  19     'libssl-dev', # This will install libssl 1.1 on Debian 11 and libssl3 on Debian 12
  20     'libsystemd-dev',
  21     'libtool',
  22     'make',
  23     'pkg-config',
  24     'python3-venv',
  25     'systemd',
  26 ]
  27 git_build_deps = [
  28     'autoconf',
  29     'automake',
  30     'bison',
  31     'bzip2',
  32     'curl',
  33     'flex',
  34     'git',
  35     'ragel'
  36 ]
  37 auth_build_deps = [    # FIXME: perhaps we should be stealing these from the debian (Ubuntu) control file
  38     'default-libmysqlclient-dev',
  39     'libcdb-dev',
  40     'libcurl4-openssl-dev',
  41     'libgeoip-dev',
  42     'libkrb5-dev',
  43     'libldap2-dev',
  44     'liblmdb-dev',
  45     'libmaxminddb-dev',
  46     'libp11-kit-dev',
  47     'libpq-dev',
  48     'libsqlite3-dev',
  49     'libyaml-cpp-dev',
  50     'libzmq3-dev',
  51     'ruby-bundler',
  52     'ruby-dev',
  53     'sqlite3',
  54     'unixodbc-dev',
  55     'cmake',
  56 ]
  57 rec_build_deps = [
  58     'libcap-dev',
  59     'libfstrm-dev',
  60     'libsnmp-dev',
  61 ]
  62 rec_bulk_deps = [
  63     'curl',
  64     'libboost-all-dev',
  65     'libcap2',
  66     'libfstrm0',
  67     'libluajit-5.1-2',
  68     '"libsnmp[1-9]+"',
  69     'libsodium23',
  70     'libsystemd0',
  71     'moreutils',
  72     'pdns-tools',
  73     'unzip',
  74 ]
  75 dnsdist_build_deps = [
  76     'libcap-dev',
  77     'libcdb-dev',
  78     'libedit-dev',
  79     'libfstrm-dev',
  80     'libgnutls28-dev',
  81     'libh2o-evloop-dev',
  82     'liblmdb-dev',
  83     'libnghttp2-dev',
  84     'libre2-dev',
  85     'libsnmp-dev',
  86 ]
  87 dnsdist_xdp_build_deps = [
  88     'libbpf-dev',
  89     'libxdp-dev',
  90 ]
  91 auth_test_deps = [   # FIXME: we should be generating some of these from shlibdeps in build
  92     'authbind',
  93     'bc',
  94     'bind9utils',
  95     'curl',
  96     'default-jre-headless',
  97     'dnsutils',
  98     'faketime',
  99     'gawk',
 100     'krb5-user',
 101     'ldnsutils',
 102     '"libboost-serialization1.7[1-9]+"',
 103     'libcdb1',
 104     'libcurl4',
 105     'libgeoip1',
 106     'libkrb5-3',
 107     '"libldap-2.[1-9]+"',
 108     'liblmdb0',
 109     'libluajit-5.1-2',
 110     'libmaxminddb0',
 111     'libnet-dns-perl',
 112     'libp11-kit0',
 113     'libpq5',
 114     'libsodium23',
 115     'libsqlite3-dev',
 116     'libsystemd0',
 117     '"libyaml-cpp0.[1-9]+"',
 118     'libzmq3-dev',
 119     'lmdb-utils',
 120     'prometheus',
 121     'ruby-bundler',
 122     'ruby-dev',
 123     'socat',
 124     'softhsm2',
 125     'unbound-host',
 126     'unixodbc',
 127     'wget',
 128 ]
 129 doc_deps = [
 130     'autoconf',
 131     'automake',
 132     'bison',
 133     'curl',
 134     'flex',
 135     'g++',
 136     'git',
 137     'latexmk',
 138     'libboost-all-dev',
 139     'libedit-dev',
 140     'libluajit-5.1-dev',
 141     'libssl-dev',
 142     'make',
 143     'pkg-config',
 144     'python3-venv',
 145     'ragel',
 146     'rsync',
 147 ]
 148 doc_deps_pdf = [
 149     'texlive-binaries',
 150     'texlive-formats-extra',
 151     'texlive-latex-extra',
 152 ]
 153
 154 @task
 155 def apt_fresh(c):
 156     c.sudo('apt-get update')
 157     c.sudo('apt-get -y --allow-downgrades dist-upgrade')
 158
 159 @task
 160 def install_clang(c):
 161     """
 162     install clang and llvm
 163     """
 164     c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version} llvm-{clang_version}')
 165
 166 @task
 167 def install_clang_tidy_tools(c):
 168     c.sudo(f'apt-get -y --no-install-recommends install clang-tidy-{clang_version} clang-tools-{clang_version} bear python3-yaml')
 169
 170 @task
 171 def install_clang_runtime(c):
 172     # this gives us the symbolizer, for symbols in asan/ubsan traces
 173     c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version}')
 174
 175 @task
 176 def ci_install_rust(c, repo):
 177     c.sudo(f'{repo}/builder-support/helpers/install_rust.sh')
 178
 179 def install_libdecaf(c, product):
 180     c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf')
 181     with c.cd('/tmp/libdecaf'):
 182         c.run('git checkout 41f349')
 183         c.run(f'CC={get_c_compiler()} CXX={get_cxx_compiler()} '
 184               'cmake -B build '
 185               '-DCMAKE_INSTALL_PREFIX=/usr/local '
 186               '-DCMAKE_INSTALL_LIBDIR=lib '
 187               '-DENABLE_STATIC=OFF '
 188               '-DENABLE_TESTS=OFF '
 189               '-DCMAKE_C_FLAGS="-Wno-sizeof-array-div -Wno-array-parameter" .')
 190         c.run('make -C build')
 191         c.run('sudo make -C build install')
 192     c.sudo(f'mkdir -p /opt/{product}/libdecaf')
 193     c.sudo(f'cp /usr/local/lib/libdecaf.so* /opt/{product}/libdecaf/.')
 194
 195 @task
 196 def install_doc_deps(c):
 197     c.sudo('apt-get install -y ' + ' '.join(doc_deps))
 198
 199 @task
 200 def install_doc_deps_pdf(c):
 201     c.sudo('apt-get install -y ' + ' '.join(doc_deps_pdf))
 202
 203 @task
 204 def install_auth_build_deps(c):
 205     c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps))
 206     if os.getenv('DECAF_SUPPORT', 'no') == 'yes':
 207         install_libdecaf(c, 'pdns-auth')
 208
 209 def is_coverage_enabled():
 210     sanitizers = os.getenv('SANITIZERS')
 211     if sanitizers:
 212         sanitizers = sanitizers.split('+')
 213         if 'tsan' in sanitizers:
 214             return False
 215     return os.getenv('COVERAGE') == 'yes'
 216
 217 def get_coverage():
 218     return '--enable-coverage=clang' if is_coverage_enabled() else ''
 219
 220 @task
 221 def install_coverage_deps(c):
 222     if is_coverage_enabled():
 223         c.sudo(f'apt-get install -y --no-install-recommends llvm-{clang_version}')
 224
 225 @task
 226 def generate_coverage_info(c, binary, outputDir):
 227     if is_coverage_enabled():
 228         version = os.getenv('BUILDER_VERSION')
 229         c.run(f'llvm-profdata-{clang_version} merge -sparse -o {outputDir}/temp.profdata /tmp/code-*.profraw')
 230         c.run(f'llvm-cov-{clang_version} export --format=lcov --ignore-filename-regex=\'^/usr/\' -instr-profile={outputDir}/temp.profdata -object {binary} > {outputDir}/coverage.lcov')
 231         c.run(f'{outputDir}/.github/scripts/normalize_paths_in_coverage.py {outputDir} {version} {outputDir}/coverage.lcov {outputDir}/normalized_coverage.lcov')
 232         c.run(f'mv {outputDir}/normalized_coverage.lcov {outputDir}/coverage.lcov')
 233
 234 def setup_authbind(c):
 235     c.sudo('touch /etc/authbind/byport/53')
 236     c.sudo('chmod 755 /etc/authbind/byport/53')
 237
 238 auth_backend_test_deps = dict(
 239     gsqlite3=['sqlite3'],
 240     gmysql=['default-libmysqlclient-dev'],
 241     gpgsql=['libpq-dev'],
 242     lmdb=[],
 243     remote=[],
 244     bind=[],
 245     geoip=[],
 246     lua2=[],
 247     tinydns=[],
 248     authpy=[],
 249     godbc_sqlite3=['libsqliteodbc'],
 250     godbc_mssql=['freetds-bin','tdsodbc'],
 251     ldap=[],
 252     geoip_mmdb=[]
 253 )
 254
 255 @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend'])
 256 def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more than apt-get
 257     extra=[]
 258     for b in backend:
 259         extra.extend(auth_backend_test_deps[b])
 260     c.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install ' + ' '.join(extra+auth_test_deps))
 261
 262     c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*')
 263     # c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then
 264     #               wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz
 265     #               tar xfz jdnssec-tools-0.14.tar.gz -C $HOME
 266     #               rm jdnssec-tools-0.14.tar.gz
 267     #          fi
 268     #          echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''')  # FIXME: why did this fail with no error?
 269     c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing regression-tests.nobackend/rectify-axfr/allow-missing') # FIXME: can this go?
 270     # FIXME we may want to start a background recursor here to make ALIAS tests more robust
 271     setup_authbind(c)
 272
 273     if os.getenv('DECAF_SUPPORT', 'no') == 'yes':
 274         # Copy libdecaf out
 275         c.sudo('mkdir -p /usr/local/lib')
 276         c.sudo('cp /opt/pdns-auth/libdecaf/libdecaf.so* /usr/local/lib/.')
 277
 278 @task
 279 def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get
 280     c.sudo('apt-get --no-install-recommends -y install ' + ' '.join(rec_bulk_deps))
 281     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 282
 283 @task
 284 def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get
 285     c.sudo('apt-get --no-install-recommends install -y ' + ' '.join(rec_bulk_deps) + ' \
 286               pdns-server pdns-backend-bind daemontools \
 287               jq libfaketime lua-posix lua-socket bc authbind \
 288               python3-venv python3-dev default-libmysqlclient-dev libpq-dev \
 289               protobuf-compiler snmpd prometheus')
 290
 291     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 292
 293     setup_authbind(c)
 294
 295     c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 296     c.sudo('/etc/init.d/snmpd restart')
 297     time.sleep(5)
 298     c.sudo('chmod 755 /var/agentx')
 299
 300 @task(optional=['skipXDP'])
 301 def install_dnsdist_test_deps(c, skipXDP=False): # FIXME: rename this, we do way more than apt-get
 302     deps = 'libluajit-5.1-2 \
 303             libboost-all-dev \
 304             libcap2 \
 305             libcdb1 \
 306             libcurl4-openssl-dev \
 307             libfstrm0 \
 308             libgnutls30 \
 309             libh2o-evloop0.13 \
 310             liblmdb0 \
 311             libnghttp2-14 \
 312             "libre2-[1-9]+" \
 313             libssl-dev \
 314             libsystemd0 \
 315             libsodium23 \
 316             lua-socket \
 317             patch \
 318             protobuf-compiler \
 319             python3-venv snmpd prometheus'
 320     if not skipXDP:
 321         deps = deps + '\
 322                libbpf1 \
 323                libxdp1'
 324
 325     c.sudo(f'apt-get install -y {deps}')
 326     c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 327     c.sudo('/etc/init.d/snmpd restart')
 328     time.sleep(5)
 329     c.sudo('chmod 755 /var/agentx')
 330
 331 @task
 332 def install_rec_build_deps(c):
 333     c.sudo('apt-get install -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + rec_build_deps))
 334
 335 @task(optional=['skipXDP'])
 336 def install_dnsdist_build_deps(c, skipXDP=False):
 337     c.sudo('apt-get install -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps + (dnsdist_xdp_build_deps if not skipXDP else [])))
 338
 339 @task
 340 def ci_autoconf(c):
 341     c.run('autoreconf -vfi')
 342
 343 @task
 344 def ci_docs_rec_generate(c):
 345     c.run('python3 generate.py')
 346
 347 @task
 348 def ci_docs_build(c):
 349     c.run('make -f Makefile.sphinx -C docs html')
 350
 351 @task
 352 def ci_docs_build_pdf(c):
 353     c.run('make -f Makefile.sphinx -C docs latexpdf')
 354
 355 @task
 356 def ci_docs_upload_master(c, docs_host, pdf, username, product, directory=""):
 357     rsync_cmd = " ".join([
 358         "rsync",
 359         "--checksum",
 360         "--recursive",
 361         "--verbose",
 362         "--no-p",
 363         "--chmod=g=rwX",
 364         "--exclude '*~'",
 365     ])
 366     c.run(f"{rsync_cmd} --delete ./docs/_build/{product}-html-docs/ {username}@{docs_host}:{directory}")
 367     c.run(f"{rsync_cmd} ./docs/_build/{product}-html-docs.tar.bz2 {username}@{docs_host}:{directory}/html-docs.tar.bz2")
 368     c.run(f"{rsync_cmd} ./docs/_build/latex/{pdf} {username}@{docs_host}:{directory}")
 369
 370 @task
 371 def ci_docs_add_ssh(c, ssh_key, host_key):
 372     c.run('mkdir -m 700 -p ~/.ssh')
 373     c.run(f'echo "{ssh_key}" > ~/.ssh/id_ed25519')
 374     c.run('chmod 600 ~/.ssh/id_ed25519')
 375     c.run(f'echo "{host_key}" > ~/.ssh/known_hosts')
 376
 377
 378 def get_sanitizers():
 379     sanitizers = os.getenv('SANITIZERS', '')
 380     if sanitizers != '':
 381         sanitizers = sanitizers.split('+')
 382         sanitizers = ['--enable-' + sanitizer for sanitizer in sanitizers]
 383         sanitizers = ' '.join(sanitizers)
 384     return sanitizers
 385
 386 def get_unit_tests(auth=False):
 387     if os.getenv('UNIT_TESTS') != 'yes':
 388         return ''
 389     return '--enable-unit-tests --enable-backend-unit-tests' if auth else '--enable-unit-tests'
 390
 391 def get_build_concurrency(default=8):
 392     return os.getenv('CONCURRENCY', default)
 393
 394 def get_fuzzing_targets():
 395     return '--enable-fuzz-targets' if os.getenv('FUZZING_TARGETS') == 'yes' else ''
 396
 397 def is_compiler_clang():
 398     compiler = os.getenv('COMPILER', 'clang')
 399     return compiler == 'clang'
 400
 401 def get_c_compiler():
 402     return f'clang-{clang_version}' if is_compiler_clang() else 'gcc'
 403
 404 def get_cxx_compiler():
 405     return f'clang++-{clang_version}' if is_compiler_clang() else 'g++'
 406
 407 def get_optimizations():
 408     optimizations = os.getenv('OPTIMIZATIONS', 'yes')
 409     return '-O1' if optimizations == 'yes' else '-O0'
 410
 411 def get_cflags():
 412     return " ".join([
 413         get_optimizations(),
 414         "-Werror=vla",
 415         "-Werror=shadow",
 416         "-Wformat=2",
 417         "-Werror=format-security",
 418         "-fstack-clash-protection",
 419         "-fstack-protector-strong",
 420         "-fcf-protection=full",
 421         "-Werror=string-plus-int" if is_compiler_clang() else '',
 422     ])
 423
 424
 425 def get_cxxflags():
 426     return " ".join([
 427         get_cflags(),
 428         "-Wp,-D_GLIBCXX_ASSERTIONS",
 429     ])
 430
 431
 432 def get_base_configure_cmd(additional_c_flags='', additional_cxx_flags='', enable_systemd=True, enable_sodium=True):
 433     cflags = " ".join([get_cflags(), additional_c_flags])
 434     cxxflags = " ".join([get_cxxflags(), additional_cxx_flags])
 435     return " ".join([
 436         f'CFLAGS="{cflags}"',
 437         f'CXXFLAGS="{cxxflags}"',
 438         './configure',
 439         f"CC='{get_c_compiler()}'",
 440         f"CXX='{get_cxx_compiler()}'",
 441         "--enable-option-checking=fatal",
 442         "--enable-systemd" if enable_systemd else '',
 443         "--with-libsodium" if enable_sodium else '',
 444         "--enable-fortify-source=auto",
 445         "--enable-auto-var-init=pattern",
 446         get_coverage(),
 447         get_sanitizers()
 448     ])
 449
 450
 451 @task
 452 def ci_auth_configure(c):
 453     unittests = get_unit_tests(True)
 454     fuzz_targets = get_fuzzing_targets()
 455     modules = " ".join([
 456         "bind",
 457         "geoip",
 458         "gmysql",
 459         "godbc",
 460         "gpgsql",
 461         "gsqlite3",
 462         "ldap",
 463         "lmdb",
 464         "lua2",
 465         "pipe",
 466         "remote",
 467         "tinydns",
 468     ])
 469     configure_cmd = " ".join([
 470         get_base_configure_cmd(),
 471         "LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib'",
 472         f"--with-modules='{modules}'",
 473         "--enable-tools",
 474         "--enable-dns-over-tls",
 475         "--enable-experimental-pkcs11",
 476         "--enable-experimental-gss-tsig",
 477         "--enable-remotebackend-zeromq",
 478         "--enable-verbose-logging",
 479         "--with-lmdb=/usr",
 480         "--with-libdecaf" if os.getenv('DECAF_SUPPORT', 'no') == 'yes' else '',
 481         "--prefix=/opt/pdns-auth",
 482         "--enable-ixfrdist",
 483         unittests,
 484         fuzz_targets
 485     ])
 486     res = c.run(configure_cmd, warn=True)
 487     if res.exited != 0:
 488         c.run('cat config.log')
 489         raise UnexpectedExit(res)
 490
 491
 492 @task
 493 def ci_rec_configure(c):
 494     unittests = get_unit_tests()
 495
 496     configure_cmd = " ".join([
 497         get_base_configure_cmd(),
 498         "--enable-nod",
 499         "--prefix=/opt/pdns-recursor",
 500         "--with-lua=luajit",
 501         "--with-libcap",
 502         "--with-net-snmp",
 503         "--enable-dns-over-tls",
 504         "--enable-verbose-logging",
 505         unittests,
 506     ])
 507     res = c.run(configure_cmd, warn=True)
 508     if res.exited != 0:
 509         c.run('cat config.log')
 510         raise UnexpectedExit(res)
 511
 512
 513 @task
 514 def ci_dnsdist_configure(c, features):
 515     additional_flags = ''
 516     if features == 'full':
 517       features_set = '--enable-dnstap \
 518                       --enable-dnscrypt \
 519                       --enable-dns-over-tls \
 520                       --enable-dns-over-https \
 521                       --enable-dns-over-quic \
 522                       --enable-dns-over-http3 \
 523                       --enable-systemd \
 524                       --prefix=/opt/dnsdist \
 525                       --with-gnutls \
 526                       --with-h2o \
 527                       --with-libsodium \
 528                       --with-lua=luajit \
 529                       --with-libcap \
 530                       --with-net-snmp \
 531                       --with-nghttp2 \
 532                       --with-re2'
 533     else:
 534       features_set = '--disable-dnstap \
 535                       --disable-dnscrypt \
 536                       --disable-ipcipher \
 537                       --disable-systemd \
 538                       --without-cdb \
 539                       --without-ebpf \
 540                       --without-gnutls \
 541                       --without-h2o \
 542                       --without-libedit \
 543                       --without-libsodium \
 544                       --without-lmdb \
 545                       --without-net-snmp \
 546                       --without-nghttp2 \
 547                       --without-re2'
 548       additional_flags = '-DDISABLE_COMPLETION \
 549                           -DDISABLE_DELAY_PIPE \
 550                           -DDISABLE_DYNBLOCKS \
 551                           -DDISABLE_PROMETHEUS \
 552                           -DDISABLE_PROTOBUF \
 553                           -DDISABLE_BUILTIN_HTML \
 554                           -DDISABLE_CARBON \
 555                           -DDISABLE_SECPOLL \
 556                           -DDISABLE_DEPRECATED_DYNBLOCK \
 557                           -DDISABLE_LUA_WEB_HANDLERS \
 558                           -DDISABLE_NON_FFI_DQ_BINDINGS \
 559                           -DDISABLE_POLICIES_BINDINGS \
 560                           -DDISABLE_PACKETCACHE_BINDINGS \
 561                           -DDISABLE_DOWNSTREAM_BINDINGS \
 562                           -DDISABLE_COMBO_ADDR_BINDINGS \
 563                           -DDISABLE_CLIENT_STATE_BINDINGS \
 564                           -DDISABLE_QPS_LIMITER_BINDINGS \
 565                           -DDISABLE_SUFFIX_MATCH_BINDINGS \
 566                           -DDISABLE_NETMASK_BINDINGS \
 567                           -DDISABLE_DNSNAME_BINDINGS \
 568                           -DDISABLE_DNSHEADER_BINDINGS \
 569                           -DDISABLE_RECVMMSG \
 570                           -DDISABLE_WEB_CACHE_MANAGEMENT \
 571                           -DDISABLE_WEB_CONFIG \
 572                           -DDISABLE_RULES_ALTERING_QUERIES \
 573                           -DDISABLE_ECS_ACTIONS \
 574                           -DDISABLE_TOP_N_BINDINGS \
 575                           -DDISABLE_OCSP_STAPLING \
 576                           -DDISABLE_HASHED_CREDENTIALS \
 577                           -DDISABLE_FALSE_SHARING_PADDING \
 578                           -DDISABLE_NPN'
 579     unittests = get_unit_tests()
 580     fuzztargets = get_fuzzing_targets()
 581     tools = f'''AR=llvm-ar-{clang_version} RANLIB=llvm-ranlib-{clang_version}''' if is_compiler_clang() else ''
 582     configure_cmd = " ".join([
 583         tools,
 584         get_base_configure_cmd(additional_c_flags='', additional_cxx_flags=additional_flags, enable_systemd=False, enable_sodium=False),
 585         features_set,
 586         unittests,
 587         fuzztargets,
 588         '--enable-lto=thin',
 589         '--prefix=/opt/dnsdist'
 590     ])
 591
 592     res = c.run(configure_cmd, warn=True)
 593     if res.exited != 0:
 594         c.run('cat config.log')
 595         raise UnexpectedExit(res)
 596
 597 @task
 598 def ci_auth_make(c):
 599     c.run(f'make -j{get_build_concurrency()} -k V=1')
 600
 601 @task
 602 def ci_auth_make_bear(c):
 603     c.run(f'bear --append -- make -j{get_build_concurrency()} -k V=1')
 604
 605 @task
 606 def ci_rec_make(c):
 607     c.run(f'make -j{get_build_concurrency()} -k V=1')
 608
 609 @task
 610 def ci_rec_make_bear(c):
 611     # Assumed to be running under ./pdns/recursordist/
 612     c.run(f'bear --append -- make -j{get_build_concurrency()} -k V=1')
 613
 614 @task
 615 def ci_dnsdist_make(c):
 616     c.run(f'make -j{get_build_concurrency(4)} -k V=1')
 617
 618 @task
 619 def ci_dnsdist_make_bear(c):
 620     # Assumed to be running under ./pdns/dnsdistdist/
 621     c.run(f'bear --append -- make -j{get_build_concurrency(4)} -k V=1')
 622
 623 @task
 624 def ci_auth_install_remotebackend_test_deps(c):
 625     with c.cd('modules/remotebackend'):
 626       # c.run('bundle config set path vendor/bundle')
 627       c.run('sudo ruby -S bundle install')
 628     c.sudo('apt-get install -y socat')
 629
 630 @task
 631 def ci_auth_run_unit_tests(c):
 632     res = c.run('make check', warn=True)
 633     if res.exited != 0:
 634       c.run('cat pdns/test-suite.log', warn=True)
 635       c.run('cat modules/remotebackend/test-suite.log', warn=True)
 636       raise UnexpectedExit(res)
 637
 638 @task
 639 def ci_rec_run_unit_tests(c):
 640     res = c.run('make check', warn=True)
 641     if res.exited != 0:
 642       c.run('cat test-suite.log')
 643       raise UnexpectedExit(res)
 644
 645 @task
 646 def ci_dnsdist_run_unit_tests(c):
 647     res = c.run('make check', warn=True)
 648     if res.exited != 0:
 649       c.run('cat test-suite.log')
 650       raise UnexpectedExit(res)
 651
 652 @task
 653 def ci_make_distdir(c):
 654     res = c.run('make distdir')
 655
 656 @task
 657 def ci_make_install(c):
 658     res = c.run('make install') # FIXME: this builds auth docs - again
 659
 660 @task
 661 def add_auth_repo(c, dist_name, dist_release_name, pdns_repo_version):
 662     c.sudo('apt-get install -y curl gnupg2')
 663     if pdns_repo_version == 'master':
 664         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc')
 665     else:
 666         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc')
 667     c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist_name} {dist_release_name}-auth-{pdns_repo_version} main' | sudo tee /etc/apt/sources.list.d/pdns.list")
 668     c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns")
 669     c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns")
 670     c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns")
 671     c.sudo('apt-get update')
 672
 673 @task
 674 def test_api(c, product, backend=''):
 675     if product == 'recursor':
 676         with c.cd('regression-tests.api'):
 677             c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}')
 678     elif product == 'auth':
 679         with c.cd('regression-tests.api'):
 680             c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432 ./runtests authoritative {backend}')
 681     else:
 682         raise Failure('unknown product')
 683
 684 backend_regress_tests = dict(
 685     bind = [
 686         'bind-both',
 687         'bind-dnssec-both',
 688         'bind-dnssec-nsec3-both',
 689         'bind-dnssec-nsec3-optout-both',
 690         'bind-dnssec-nsec3-narrow',
 691         'bind-dnssec-pkcs11'
 692     ],
 693     geoip = [
 694         'geoip',
 695         'geoip-nsec3-narrow'
 696     ],
 697     lua2 = ['lua2', 'lua2-dnssec'],
 698     tinydns = ['tinydns'],
 699     remote = [
 700         'remotebackend-pipe',
 701         'remotebackend-unix',
 702         'remotebackend-http',
 703         'remotebackend-zeromq',
 704         'remotebackend-pipe-dnssec',
 705         'remotebackend-unix-dnssec',
 706         'remotebackend-http-dnssec',
 707         'remotebackend-zeromq-dnssec'
 708     ],
 709     lmdb = [
 710         'lmdb-nodnssec-both',
 711         'lmdb-both',
 712         'lmdb-nsec3-both',
 713         'lmdb-nsec3-optout-both',
 714         'lmdb-nsec3-narrow'
 715     ],
 716     gmysql = [
 717         'gmysql',
 718         'gmysql-nodnssec-both',
 719         'gmysql-nsec3-both',
 720         'gmysql-nsec3-optout-both',
 721         'gmysql-nsec3-narrow',
 722         'gmysql_sp-both'
 723     ],
 724     gpgsql = [
 725         'gpgsql',
 726         'gpgsql-nodnssec-both',
 727         'gpgsql-nsec3-both',
 728         'gpgsql-nsec3-optout-both',
 729         'gpgsql-nsec3-narrow',
 730         'gpgsql_sp-both'
 731     ],
 732     gsqlite3 = [
 733         'gsqlite3',
 734         'gsqlite3-nodnssec-both',
 735         'gsqlite3-nsec3-both',
 736         'gsqlite3-nsec3-optout-both',
 737         'gsqlite3-nsec3-narrow'
 738     ],
 739     godbc_sqlite3 = ['godbc_sqlite3-nodnssec'],
 740     godbc_mssql = [
 741         'godbc_mssql',
 742         'godbc_mssql-nodnssec',
 743         'godbc_mssql-nsec3',
 744         'godbc_mssql-nsec3-optout',
 745         'godbc_mssql-nsec3-narrow'
 746     ],
 747     ldap = [
 748         'ldap-tree',
 749         'ldap-simple',
 750         'ldap-strict'
 751     ],
 752     geoip_mmdb = ['geoip'],
 753 )
 754
 755 godbc_mssql_credentials = {"username": "sa", "password": "SAsa12%%-not-a-secret-password"}
 756
 757 godbc_config = f'''
 758 [pdns-mssql-docker]
 759 Driver=FreeTDS
 760 Trace=No
 761 Server={auth_backend_ip_addr}
 762 Port=1433
 763 Database=pdns
 764 TDS_Version=7.1
 765
 766 [pdns-mssql-docker-nodb]
 767 Driver=FreeTDS
 768 Trace=No
 769 Server={auth_backend_ip_addr}
 770 Port=1433
 771 TDS_Version=7.1
 772
 773 [pdns-sqlite3-1]
 774 Driver = SQLite3
 775 Database = pdns.sqlite3
 776
 777 [pdns-sqlite3-2]
 778 Driver = SQLite3
 779 Database = pdns.sqlite32
 780 '''
 781
 782 def setup_godbc_mssql(c):
 783     with open(os.path.expanduser("~/.odbc.ini"), "a") as f:
 784         f.write(godbc_config)
 785     c.sudo('sh -c \'echo "Threading=1" | cat /usr/share/tdsodbc/odbcinst.ini - | tee -a /etc/odbcinst.ini\'')
 786     c.sudo('sed -i "s/libtdsodbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libtdsodbc.so/g" /etc/odbcinst.ini')
 787     c.run(f'echo "create database pdns" | isql -v pdns-mssql-docker-nodb {godbc_mssql_credentials["username"]} {godbc_mssql_credentials["password"]}')
 788     # FIXME: Skip 8bit-txt-unescaped test
 789     c.run('touch ${PWD}/regression-tests/tests/8bit-txt-unescaped/skip')
 790
 791 def setup_godbc_sqlite3(c):
 792     with open(os.path.expanduser("~/.odbc.ini"), "a") as f:
 793         f.write(godbc_config)
 794     c.sudo('sed -i "s/libsqlite3odbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libsqlite3odbc.so/g" /etc/odbcinst.ini')
 795
 796 def setup_ldap_client(c):
 797     c.sudo('DEBIAN_FRONTEND=noninteractive apt-get install -y ldap-utils')
 798     c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} ldapserver" | tee -a /etc/hosts\'')
 799
 800 def setup_softhsm(c):
 801     # Modify the location of the softhsm tokens and configuration directory.
 802     # Enables token generation by non-root users (runner)
 803     c.run('mkdir -p /opt/pdns-auth/softhsm/tokens')
 804     c.run('echo "directories.tokendir = /opt/pdns-auth/softhsm/tokens" > /opt/pdns-auth/softhsm/softhsm2.conf')
 805
 806 @task
 807 def test_auth_backend(c, backend):
 808     pdns_auth_env_vars = f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST={auth_backend_ip_addr} GMYSQL2HOST={auth_backend_ip_addr} MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432'
 809
 810     if backend == 'remote':
 811         ci_auth_install_remotebackend_test_deps(c)
 812
 813     if backend == 'authpy':
 814         c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} kerberos-server" | tee -a /etc/hosts\'')
 815         with c.cd('regression-tests.auth-py'):
 816             c.run(f'{pdns_auth_env_vars} WITHKERBEROS=YES ./runtests')
 817         return
 818
 819     if backend == 'bind':
 820         setup_softhsm(c)
 821         with c.cd('regression-tests'):
 822             for variant in backend_regress_tests[backend]:
 823                 c.run(f'{pdns_auth_env_vars} SOFTHSM2_CONF=/opt/pdns-auth/softhsm/softhsm2.conf ./start-test-stop 5300 {variant}')
 824         return
 825
 826     if backend == 'godbc_sqlite3':
 827         setup_godbc_sqlite3(c)
 828         with c.cd('regression-tests'):
 829             for variant in backend_regress_tests[backend]:
 830                 c.run(f'{pdns_auth_env_vars} GODBC_SQLITE3_DSN=pdns-sqlite3-1 ./start-test-stop 5300 {variant}')
 831         return
 832
 833     if backend == 'godbc_mssql':
 834         setup_godbc_mssql(c)
 835         with c.cd('regression-tests'):
 836             for variant in backend_regress_tests[backend]:
 837                 c.run(f'{pdns_auth_env_vars} GODBC_MSSQL_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL_DSN=pdns-mssql-docker GODBC_MSSQL2_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL2_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL2_DSN=pdns-mssql-docker ./start-test-stop 5300 {variant}')
 838         return
 839
 840     if backend == 'ldap':
 841         setup_ldap_client(c)
 842
 843     if backend == 'geoip_mmdb':
 844         with c.cd('regression-tests'):
 845             for variant in backend_regress_tests[backend]:
 846                 c.run(f'{pdns_auth_env_vars} geoipdatabase=../modules/geoipbackend/regression-tests/GeoLiteCity.mmdb ./start-test-stop 5300 {variant}')
 847         return
 848
 849     with c.cd('regression-tests'):
 850         if backend == 'lua2':
 851             c.run('touch trustedkeys')  # avoid silly error during cleanup
 852         for variant in backend_regress_tests[backend]:
 853             c.run(f'{pdns_auth_env_vars} ./start-test-stop 5300 {variant}')
 854
 855     if backend == 'gsqlite3':
 856         if os.getenv('SKIP_IPV6_TESTS'):
 857             pdns_auth_env_vars += ' context=noipv6'
 858         with c.cd('regression-tests.nobackend'):
 859             c.run(f'{pdns_auth_env_vars} ./runtests')
 860         c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms')
 861         return
 862
 863 @task
 864 def test_ixfrdist(c):
 865     with c.cd('regression-tests.ixfrdist'):
 866         c.run('IXFRDISTBIN=/opt/pdns-auth/bin/ixfrdist ./runtests')
 867
 868 @task
 869 def test_dnsdist(c):
 870     c.run('chmod +x /opt/dnsdist/bin/*')
 871     c.run('ls -ald /var /var/agentx /var/agentx/master')
 872     c.run('ls -al /var/agentx/master')
 873     with c.cd('regression-tests.dnsdist'):
 874         c.run('DNSDISTBIN=/opt/dnsdist/bin/dnsdist LD_LIBRARY_PATH=/opt/dnsdist/lib/ ENABLE_SUDO_TESTS=1 ./runtests')
 875
 876 @task
 877 def test_regression_recursor(c):
 878     c.run('/opt/pdns-recursor/sbin/pdns_recursor --version')
 879     c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control ./build-scripts/test-recursor')
 880
 881 @task
 882 def test_bulk_recursor(c, threads, mthreads, shards):
 883     # We run an extremely small version of the bulk test, as GH does not seem to be able to handle the UDP load
 884     with c.cd('regression-tests'):
 885         c.run('curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip')
 886         c.run('unzip top-1m.csv.zip -d .')
 887         c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 888         c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control THRESHOLD=95 TRACE=no ./recursor-test 5300 100 {threads} {mthreads} {shards}')
 889
 890 @task
 891 def install_swagger_tools(c):
 892     c.run('npm install -g api-spec-converter')
 893
 894 @task
 895 def swagger_syntax_check(c):
 896     c.run('api-spec-converter docs/http-api/swagger/authoritative-api-swagger.yaml -f swagger_2 -t openapi_3 -s json -c')
 897
 898 @task
 899 def install_coverity_tools(c, project):
 900     token = os.getenv('COVERITY_TOKEN')
 901     c.run(f'curl -s https://scan.coverity.com/download/linux64 --data "token={token}&project={project}" | gunzip | sudo tar xvf /dev/stdin --strip-components=1 --no-same-owner -C /usr/local', hide=True)
 902
 903 @task
 904 def coverity_clang_configure(c):
 905     c.sudo(f'/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-{clang_version}')
 906
 907 @task
 908 def coverity_make(c):
 909     c.run('/usr/local/bin/cov-build --dir cov-int make -j8 -k')
 910
 911 @task
 912 def coverity_tarball(c, tarball):
 913     c.run(f'tar caf {tarball} cov-int')
 914
 915 @task
 916 def coverity_upload(c, email, project, tarball):
 917     token = os.getenv('COVERITY_TOKEN')
 918     c.run(f'curl --form token={token} \
 919             --form email="{email}" \
 920             --form file=@{tarball} \
 921             --form version="$(./builder-support/gen-version)" \
 922             --form description="master build" \
 923             https://scan.coverity.com/builds?project={project}', hide=True)
 924
 925 @task
 926 def ci_build_and_install_quiche(c):
 927     # we have to pass -L because GitHub will do a redirect, sadly
 928     c.run(f'curl -L -o quiche-{quiche_version}.tar.gz https://github.com/cloudflare/quiche/archive/{quiche_version}.tar.gz')
 929     # Line below should echo two spaces between digest and name
 930     c.run(f'echo {quiche_hash}"  "quiche-{quiche_version}.tar.gz | sha256sum -c -')
 931     c.run(f'tar xf quiche-{quiche_version}.tar.gz')
 932     with c.cd(f'quiche-{quiche_version}'):
 933         c.run('cargo build --release --no-default-features --features ffi,boringssl-boring-crate --package quiche')
 934         # cannot use c.sudo() inside a cd() context, see https://github.com/pyinvoke/invoke/issues/687
 935         c.run('sudo install -Dm644 quiche/include/quiche.h /usr/include')
 936         c.run('sudo install -Dm644 target/release/libquiche.so /usr/lib')
 937         c.run('install -D target/release/libquiche.so /opt/dnsdist/lib/libquiche.so')
 938         c.run(f"""sudo install -Dm644 /dev/stdin /usr/lib/pkgconfig/quiche.pc <<PC
 939 # quiche
 940 Name: quiche
 941 Description: quiche library
 942 URL: https://github.com/cloudflare/quiche
 943 Version: {quiche_version}
 944 Libs: -lquiche
 945 PC""")
 946
 947 # this is run always
 948 def setup():
 949     if '/usr/lib/ccache' not in os.environ['PATH']:
 950         os.environ['PATH']='/usr/lib/ccache:'+os.environ['PATH']
 951
 952 setup()