From 0e77de07598d014955aa2652d4ad60222b0eebf9 Mon Sep 17 00:00:00 2001 From: Peter van Dijk Date: Tue, 24 Aug 2021 11:35:27 +0200 Subject: [PATCH] run auth remotebackend regression tests in GH Actions; remove from CircleCI --- .circleci/config.yml | 46 ---------------------- .github/workflows/build-and-test-all.yml | 50 ++++++++++++++++++++++++ tasks.py | 24 ++++++++++-- 3 files changed, 71 insertions(+), 49 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af0ebf185d..ee87715f5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -959,49 +959,6 @@ jobs: context: geoip doroot: false - test-auth-regress-remote: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - auth-regress-setup - - run: apt-get -y -qq install ruby ruby-bundler ruby2.5-dev - - run: - workdir: ~/project/modules/remotebackend - command: ruby -S bundle install - - auth-regress: - context: remotebackend-pipe - doroot: false - - auth-regress: - context: remotebackend-unix - doroot: false - - auth-regress: - context: remotebackend-http - doroot: false - - auth-regress: - context: remotebackend-zeromq - doroot: false - - auth-regress: - context: remotebackend-pipe-dnssec - doroot: false - - auth-regress: - context: remotebackend-unix-dnssec - doroot: false - - auth-regress: - context: remotebackend-http-dnssec - doroot: false - - auth-regress: - context: remotebackend-zeromq-dnssec - doroot: false - - test-auth-regress-lua2: resource_class: small @@ -1975,9 +1932,6 @@ workflows: - test-auth-regress-geoip: requires: - build-auth - - test-auth-regress-remote: - requires: - - build-auth - test-auth-regress-lua2: requires: - build-auth diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml index d82fd70a39..59aafd3abe 100644 --- a/.github/workflows/build-and-test-all.yml +++ b/.github/workflows/build-and-test-all.yml @@ -64,6 +64,7 @@ jobs: with: name: pdns-recursor path: /opt/pdns-recursor + build-dnsdist: name: build dnsdist runs-on: ubuntu-20.04 @@ -142,6 +143,55 @@ jobs: - run: inv install-auth-test-deps -b ${{ matrix.backend }} - run: inv test-api auth -b ${{ matrix.backend }} + test-auth-backend: + needs: build-auth + runs-on: ubuntu-20.04 + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + strategy: + matrix: + include: + - backend: remote + image: coscale/docker-sleep + # - backend: gmysql + # image: mysql:5 + # - backend: gpgsql + # image: postgres:9 + # - backend: lmdb + # image: coscale/docker-sleep + fail-fast: false + services: + database: + image: ${{ matrix.image }} + env: + POSTGRES_USER: runner + POSTGRES_HOST_AUTH_METHOD: trust + MYSQL_ALLOW_EMPTY_PASSWORD: 1 + ports: + - 3306:3306 + - 5432:5432 + # FIXME: this works around dist-upgrade stopping all docker containers. dist-upgrade is huge on these images anyway. Perhaps we do want to run our tasks in a Docker container too. + options: >- + --restart always + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: recursive + - name: Fetch the binaries + uses: actions/download-artifact@v2 + with: + name: pdns-auth + path: /opt/pdns-auth + # - name: Setup upterm session + # uses: lhotari/action-upterm@v1 + # FIXME: install recursor for backends that have ALIAS + - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv install-clang-runtime + - run: inv install-auth-test-deps -b ${{ matrix.backend }} + - run: inv test-auth-backend -b ${{ matrix.backend }} + test-recursor-api: needs: build-recursor runs-on: ubuntu-20.04 diff --git a/tasks.py b/tasks.py index 221e764b0f..06408040fe 100644 --- a/tasks.py +++ b/tasks.py @@ -87,6 +87,8 @@ auth_test_deps = [ # FIXME: we should be generating some of these from shlibde 'libyaml-cpp0.6', 'libzmq3-dev', 'pdns-recursor', + 'ruby-bundler', + 'ruby-dev', 'socat', 'softhsm2', 'unbound-host', @@ -123,7 +125,8 @@ auth_backend_test_deps = dict( gsqlite3=['sqlite3'], gmysql=['default-libmysqlclient-dev'], gpgsql=['libpq-dev'], - lmdb=[] + lmdb=[], + remote=[] ) @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend']) @@ -282,8 +285,8 @@ def ci_dnsdist_make(c): @task def ci_auth_install_remotebackend_ruby_deps(c): with c.cd('modules/remotebackend'): - c.run('bundle config set path vendor/bundle') - c.run('ruby -S bundle install') + # c.run('bundle config set path vendor/bundle') + c.run('sudo ruby -S bundle install') @task def ci_auth_run_unit_tests(c): @@ -338,6 +341,21 @@ def test_api(c, product, backend=''): else: raise Failure('unknown product') +backend_regress_tests = dict( + remote = ['pipe', 'unix', 'http', 'zeromq', 'pipe-dnssec', 'unix-dnssec', 'http-dnssec', 'zeromq-dnssec'] +) + +@task +def test_auth_backend(c, backend): + if backend == 'remote': + ci_auth_install_remotebackend_ruby_deps(c) + + with c.cd('regression-tests'): + for t in backend_regress_tests[backend]: + # FIXME this long line is terrible + # FIXME this appends 'backend' but that's only correct for 'remote' + c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./start-test-stop 5300 {backend}backend-{t}') + @task def test_dnsdist(c): c.run('chmod +x /opt/dnsdist/bin/*') -- 2.47.2