tasks.py

   1 from invoke import task
   2 from invoke.exceptions import Failure, UnexpectedExit
   3
   4 import os
   5 import sys
   6 import time
   7
   8 all_build_deps = [
   9     'ccache',
  10     'libboost-all-dev',
  11     'libluajit-5.1-dev',
  12     'libsodium-dev',
  13     'libssl-dev',
  14     'libsystemd-dev',
  15     'libtool',
  16     'make',
  17     'pkg-config',
  18     'python3-venv',
  19     'systemd',
  20 ]
  21 git_build_deps = [
  22     'autoconf',
  23     'automake',
  24     'bison',
  25     'bzip2',
  26     'curl',
  27     'flex',
  28     'git',
  29     'ragel'
  30 ]
  31 auth_build_deps = [    # FIXME: perhaps we should be stealing these from the debian (Ubuntu) control file
  32     'default-libmysqlclient-dev',
  33     'libcdb-dev',
  34     'libcurl4-openssl-dev',
  35     'libgeoip-dev',
  36     'libkrb5-dev',
  37     'libldap2-dev',
  38     'liblmdb-dev',
  39     'libmaxminddb-dev',
  40     'libp11-kit-dev',
  41     'libpq-dev',
  42     'libsqlite3-dev',
  43     'libyaml-cpp-dev',
  44     'libzmq3-dev',
  45     'ruby-bundler',
  46     'ruby-dev',
  47     'sqlite3',
  48 ]
  49 rec_build_deps = [
  50     'libcap-dev',
  51     'libfstrm-dev',
  52     'libsnmp-dev',
  53 ]
  54 rec_bulk_deps = [
  55     'curl',
  56     'libboost-all-dev',
  57     'libcap2',
  58     'libfstrm0',
  59     'libluajit-5.1-2',
  60     'libsnmp35',
  61     'libsodium23',
  62     'libssl1.1',
  63     'libsystemd0',
  64     'moreutils',
  65     'pdns-tools',
  66     'unzip'
  67 ]
  68 dnsdist_build_deps = [
  69     'libcap-dev',
  70     'libcdb-dev',
  71     'libedit-dev',
  72     'libfstrm-dev',
  73     'libgnutls28-dev',
  74     'libh2o-evloop-dev',
  75     'liblmdb-dev',
  76     'libnghttp2-dev',
  77     'libre2-dev',
  78     'libsnmp-dev',
  79 ]
  80 auth_test_deps = [   # FIXME: we should be generating some of these from shlibdeps in build
  81     'authbind',
  82     'bc',
  83     'bind9utils',
  84     'curl',
  85     'default-jre-headless',
  86     'dnsutils',
  87     'faketime',
  88     'gawk',
  89     'ldnsutils',
  90     'libboost-serialization1.71.0',
  91     'libcdb1',
  92     'libcurl4',
  93     'libgeoip1',
  94     'libkrb5-3',
  95     'libldap-2.4-2',
  96     'liblmdb0',
  97     'libluajit-5.1-2',
  98     'libmaxminddb0',
  99     'libnet-dns-perl',
 100     'libp11-kit0',
 101     'libpq5',
 102     'libsodium23',
 103     'libsqlite3-dev',
 104     'libssl1.1',
 105     'libsystemd0',
 106     'libyaml-cpp0.6',
 107     'libzmq3-dev',
 108     'ruby-bundler',
 109     'ruby-dev',
 110     'socat',
 111     'softhsm2',
 112     'unbound-host',
 113     'unixodbc',
 114     'wget'
 115 ]
 116
 117 @task
 118 def apt_fresh(c):
 119     c.sudo('apt-get update')
 120     c.sudo('apt-get dist-upgrade')
 121
 122 @task
 123 def install_clang(c):
 124     """
 125     install clang-12 and llvm-12
 126     """
 127     c.sudo('apt-get -qq -y --no-install-recommends install clang-12 llvm-12')
 128
 129 @task
 130 def install_clang_runtime(c):
 131     # this gives us the symbolizer, for symbols in asan/ubsan traces
 132     c.sudo('apt-get -qq -y --no-install-recommends install clang-12')
 133
 134 @task
 135 def install_auth_build_deps(c):
 136     c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps))
 137
 138 def setup_authbind(c):
 139     c.sudo('touch /etc/authbind/byport/53')
 140     c.sudo('chmod 755 /etc/authbind/byport/53')
 141
 142 auth_backend_test_deps = dict(
 143     gsqlite3=['sqlite3'],
 144     gmysql=['default-libmysqlclient-dev'],
 145     gpgsql=['libpq-dev'],
 146     lmdb=[],
 147     remote=[],
 148     bind=[],
 149     geoip=[],
 150     lua2=[],
 151     tinydns=[],
 152     authpy=[]
 153 )
 154
 155 @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend'])
 156 def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more than apt-get
 157     extra=[]
 158     for b in backend:
 159         extra.extend(auth_backend_test_deps[b])
 160     c.sudo('apt-get -y -qq install ' + ' '.join(extra+auth_test_deps))
 161
 162     c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*')
 163     # c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then
 164     #               wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz
 165     #               tar xfz jdnssec-tools-0.14.tar.gz -C $HOME
 166     #               rm jdnssec-tools-0.14.tar.gz
 167     #          fi
 168     #          echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''')  # FIXME: why did this fail with no error?
 169     c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing regression-tests.nobackend/rectify-axfr/allow-missing') # FIXME: can this go?
 170     # FIXME we may want to start a background recursor here to make ALIAS tests more robust
 171     setup_authbind(c)
 172
 173 @task
 174 def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get
 175     c.sudo('apt-get --no-install-recommends -qq -y install ' + ' '.join(rec_bulk_deps))
 176     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 177
 178 @task
 179 def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get
 180     c.sudo('apt-get --no-install-recommends install -qq -y ' + ' '.join(rec_bulk_deps) + ' \
 181               pdns-server pdns-backend-bind daemontools \
 182               jq libfaketime lua-posix lua-socket bc authbind \
 183               python3-venv python3-dev default-libmysqlclient-dev libpq-dev \
 184               protobuf-compiler snmpd prometheus')
 185
 186     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 187
 188     setup_authbind(c)
 189
 190     c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 191     c.sudo('systemctl restart snmpd')
 192     time.sleep(5)
 193     c.sudo('chmod 755 /var/agentx')
 194
 195 @task
 196 def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt-get
 197     c.sudo('apt-get install -qq -y \
 198               libluajit-5.1-2 \
 199               libboost-all-dev \
 200               libcap2 \
 201               libcdb1 \
 202               libcurl4-openssl-dev \
 203               libfstrm0 \
 204               libgnutls30 \
 205               libh2o-evloop0.13 \
 206               liblmdb0 \
 207               libnghttp2-14 \
 208               libre2-5 \
 209               libssl-dev \
 210               libsystemd0 \
 211               libsodium23 \
 212               patch \
 213               protobuf-compiler \
 214               python3-venv snmpd prometheus')
 215     c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf')
 216     c.sudo('systemctl restart snmpd')
 217     time.sleep(5)
 218     c.sudo('chmod 755 /var/agentx')
 219
 220 @task
 221 def install_rec_build_deps(c):
 222     c.sudo('apt-get install -qq -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + rec_build_deps))
 223
 224 @task
 225 def install_dnsdist_build_deps(c):
 226     c.sudo('apt-get install -qq -y --no-install-recommends ' +  ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps))
 227
 228 @task
 229 def ci_autoconf(c):
 230     c.run('BUILDER_VERSION=0.0.0-git1 autoreconf -vfi')
 231
 232 @task
 233 def ci_auth_configure(c):
 234     res = c.run('''CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
 235                    CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
 236                    ./configure \
 237                       CC='clang-12' \
 238                       CXX='clang++-12' \
 239                       --enable-option-checking=fatal \
 240                       --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' \
 241                       --enable-systemd \
 242                       --enable-tools \
 243                       --enable-unit-tests \
 244                       --enable-backend-unit-tests \
 245                       --enable-fuzz-targets \
 246                       --enable-experimental-pkcs11 \
 247                       --enable-remotebackend-zeromq \
 248                       --with-lmdb=/usr \
 249                       --with-libsodium \
 250                       --prefix=/opt/pdns-auth \
 251                       --enable-ixfrdist \
 252                       --enable-asan \
 253                       --enable-ubsan''', warn=True)
 254     if res.exited != 0:
 255         c.run('cat config.log')
 256         raise UnexpectedExit(res)
 257 @task
 258 def ci_rec_configure(c):
 259     res = c.run('''            CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
 260             CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
 261             ./configure \
 262               CC='clang-12' \
 263               CXX='clang++-12' \
 264               --enable-option-checking=fatal \
 265               --enable-unit-tests \
 266               --enable-nod \
 267               --enable-systemd \
 268               --prefix=/opt/pdns-recursor \
 269               --with-libsodium \
 270               --with-lua=luajit \
 271               --with-libcap \
 272               --with-net-snmp \
 273               --enable-dns-over-tls \
 274               --enable-asan \
 275               --enable-ubsan''', warn=True)
 276     if res.exited != 0:
 277         c.run('cat config.log')
 278         raise UnexpectedExit(res)
 279
 280 @task
 281 def ci_dnsdist_configure(c):
 282     sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+'))
 283     res = c.run('''CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
 284                    CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
 285                    ./configure \
 286                      CC='clang-12' \
 287                      CXX='clang++-12' \
 288                      --enable-option-checking=fatal \
 289                      --enable-unit-tests \
 290                      --enable-dnstap \
 291                      --enable-dnscrypt \
 292                      --enable-dns-over-tls \
 293                      --enable-dns-over-https \
 294                      --enable-systemd \
 295                      --prefix=/opt/dnsdist \
 296                      --with-gnutls \
 297                      --with-libsodium \
 298                      --with-lua=luajit \
 299                      --with-libcap \
 300                      --with-nghttp2 \
 301                      --with-re2 ''' + sanitizers, warn=True)
 302     if res.exited != 0:
 303         c.run('cat config.log')
 304         raise UnexpectedExit(res)
 305
 306 @task
 307 def ci_auth_make(c):
 308     c.run('make -j8 -k V=1')
 309
 310 @task
 311 def ci_rec_make(c):
 312     c.run('make -j8 -k V=1')
 313
 314 @task
 315 def ci_dnsdist_make(c):
 316     c.run('make -j4 -k V=1')
 317
 318 @task
 319 def ci_auth_install_remotebackend_ruby_deps(c):
 320     with c.cd('modules/remotebackend'):
 321       # c.run('bundle config set path vendor/bundle')
 322       c.run('sudo ruby -S bundle install')
 323
 324 @task
 325 def ci_auth_run_unit_tests(c):
 326     res = c.run('make check', warn=True)
 327     if res.exited != 0:
 328       c.run('cat pdns/test-suite.log', warn=True)
 329       c.run('cat modules/remotebackend/test-suite.log', warn=True)
 330       raise UnexpectedExit(res)
 331
 332 @task
 333 def ci_rec_run_unit_tests(c):
 334     res = c.run('make check', warn=True)
 335     if res.exited != 0:
 336       c.run('cat test-suite.log')
 337       raise UnexpectedExit(res)
 338
 339 @task
 340 def ci_dnsdist_run_unit_tests(c):
 341     res = c.run('make check', warn=True)
 342     if res.exited != 0:
 343       c.run('cat test-suite.log')
 344       raise UnexpectedExit(res)
 345
 346 @task
 347 def ci_make_install(c):
 348     res = c.run('make install') # FIXME: this builds auth docs - again
 349
 350 @task
 351 def add_auth_repo(c):
 352     dist = 'ubuntu' # FIXME take these from the caller?
 353     release = 'focal'
 354     version = '44'
 355
 356     c.sudo('apt-get install -qq -y curl gnupg2')
 357     if version == 'master':
 358         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc')
 359     else:
 360         c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc')
 361     c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist} {release}-auth-{version} main' | sudo tee /etc/apt/sources.list.d/pdns.list")
 362     c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns")
 363     c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns")
 364     c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns")
 365     c.sudo('apt-get update')
 366
 367 @task
 368 def test_api(c, product, backend=''):
 369     if product == 'recursor':
 370         with c.cd('regression-tests.api'):
 371             c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}')
 372     elif product == 'auth':
 373         with c.cd('regression-tests.api'):
 374             c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests authoritative {backend}')
 375     else:
 376         raise Failure('unknown product')
 377
 378 backend_regress_tests = dict(
 379     bind = [
 380       'bind-both',
 381       'bind-dnssec-both',
 382       'bind-dnssec-nsec3-both',
 383       'bind-dnssec-nsec3-optout-both',
 384       'bind-dnssec-nsec3-narrow',
 385     # FIXME  'bind-dnssec-pkcs11'
 386     ],
 387     geoip = [
 388       'geoip',
 389       'geoip-nsec3-narrow'
 390       # FIXME: also run this with the mmdb we ship
 391     ],
 392     lua2 = [
 393       'lua2',
 394       'lua2-dnssec'
 395     ],
 396     tinydns = [
 397       'tinydns'
 398     ],
 399     remote = [
 400       'remotebackend-pipe',
 401       'remotebackend-unix',
 402       'remotebackend-http',
 403       'remotebackend-zeromq',
 404       'remotebackend-pipe-dnssec',
 405       'remotebackend-unix-dnssec',
 406       'remotebackend-http-dnssec',
 407       'remotebackend-zeromq-dnssec'
 408     ],
 409     lmdb = [
 410       'lmdb-nodnssec-both',
 411       'lmdb-both',
 412       'lmdb-nsec3-both',
 413       'lmdb-nsec3-optout-both',
 414       'lmdb-nsec3-narrow'
 415     ],
 416     gmysql   = ['gmysql',     'gmysql-nodnssec-both',   'gmysql-nsec3-both',   'gmysql-nsec3-optout-both',   'gmysql-nsec3-narrow',   'gmysql_sp-both'],
 417     gpgsql   = ['gpgsql',     'gpgsql-nodnssec-both',   'gpgsql-nsec3-both',   'gpgsql-nsec3-optout-both',   'gpgsql-nsec3-narrow',   'gpgsql_sp-both'],
 418     gsqlite3 = ['gsqlite3', 'gsqlite3-nodnssec-both', 'gsqlite3-nsec3-both', 'gsqlite3-nsec3-optout-both', 'gsqlite3-nsec3-narrow'],
 419 )
 420
 421 @task
 422 def test_auth_backend(c, backend):
 423     if backend == 'remote':
 424         ci_auth_install_remotebackend_ruby_deps(c)
 425
 426     if backend == 'authpy':
 427         with c.cd('regression-tests.auth-py'):
 428             c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests')
 429         return
 430
 431     with c.cd('regression-tests'):
 432         if backend == 'lua2':
 433             c.run('touch trustedkeys')  # avoid silly error during cleanup
 434         for variant in backend_regress_tests[backend]:
 435             # FIXME this long line is terrible
 436             c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./start-test-stop 5300 {variant}')
 437
 438     if backend == 'gsqlite3':
 439         with c.cd('regression-tests.nobackend'):
 440             c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests')
 441         c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms')
 442         return
 443
 444 @task
 445 def test_ixfrdist(c):
 446     with c.cd('regression-tests.ixfrdist'):
 447         c.run('IXFRDISTBIN=/opt/pdns-auth/bin/ixfrdist ./runtests')
 448
 449 @task
 450 def test_dnsdist(c):
 451     c.run('chmod +x /opt/dnsdist/bin/*')
 452     c.run('ls -ald /var /var/agentx /var/agentx/master')
 453     c.run('ls -al /var/agentx/master')
 454     with c.cd('regression-tests.dnsdist'):
 455         c.run('DNSDISTBIN=/opt/dnsdist/bin/dnsdist ./runtests')
 456
 457 @task
 458 def test_regression_recursor(c):
 459     c.run('/opt/pdns-recursor/sbin/pdns_recursor --version')
 460     c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control SKIP_IPV6_TESTS=y LIBFAKETIME=/bin/false ./build-scripts/test-recursor test_RecDnstap.py')
 461     c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control SKIP_IPV6_TESTS=y ./build-scripts/test-recursor -I test_RecDnstap.py')
 462
 463 @task
 464 def test_bulk_recursor(c, threads, mthreads, shards):
 465     # We run an extremely small version of the bulk test, as GH does not seem to be able to handle the UDP load
 466     with c.cd('regression-tests'):
 467         c.run('curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip')
 468         c.run('unzip top-1m.csv.zip -d .')
 469         c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 470         c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control THRESHOLD=95 TRACE=no ./timestamp ./recursor-test 5300 100 {threads} {mthreads} {shards}')
 471
 472 # this is run always
 473 def setup():
 474     if '/usr/lib/ccache' not in os.environ['PATH']:
 475         os.environ['PATH']='/usr/lib/ccache:'+os.environ['PATH']
 476
 477 setup()