From: Peter van Dijk Date: Fri, 8 Oct 2021 11:23:48 +0000 (+0200) Subject: gh actions: ixfrdist, auth-algorithms, auth-py, auth gsqlite3, auth-nobackend X-Git-Tag: dnsdist-1.7.0-alpha2~7^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F10827%2Fhead;p=thirdparty%2Fpdns.git gh actions: ixfrdist, auth-algorithms, auth-py, auth gsqlite3, auth-nobackend also, remove dnsdist from CircleCI as it's already fully covered in GH Actions now --- diff --git a/.circleci/config.yml b/.circleci/config.yml index 71acd24a90..5773900220 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -251,20 +251,6 @@ commands: protobuf-compiler \ python3-venv - install-ixfrdist-deps: - description: "Install all libraries needed for testing ixfrdist" - steps: - - run: apt-get update - - run: - command: | - apt-get install -qq -y \ - git \ - libboost-all-dev \ - libsodium23 \ - libsystemd0 \ - libyaml-cpp0.6 \ - python3-venv - install-auth-dev-deps: description: Install all packages needed to build the auth steps: @@ -817,47 +803,6 @@ jobs: context: geoip doroot: false - test-auth-algorithms: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - auth-regress-setup - - run: - name: Test all algorithms - command: /opt/pdns-auth/bin/pdnsutil test-algorithms - - test-auth-py: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - auth-regress-setup - - run: apt-get -y -qq install python3-venv - - run: - name: Run auth-py tests - workdir: ~/project/regression-tests.auth-py - command: | - PDNS="/opt/pdns-auth/sbin/pdns_server" \ - PDNSUTIL="/opt/pdns-auth/bin/pdnsutil" \ - SDIG="/opt/pdns-auth/bin/sdig" \ - ZONE2SQL="/opt/pdns-auth/bin/zone2sql" \ - ./runtests - build-recursor: docker: - image: debian:buster @@ -1480,118 +1425,6 @@ jobs: --form description="master build" \ https://scan.coverity.com/builds?project=${COVERITY_PROJECT} - build-dnsdist: - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - run: - name: Install dependencies - command: | - apt-get update && apt-get -qq --no-install-recommends install \ - autoconf \ - automake \ - g++ \ - git \ - libboost-all-dev \ - libcap-dev \ - libcdb-dev \ - libedit-dev \ - libfstrm-dev \ - libgnutls28-dev \ - libh2o-evloop-dev \ - liblmdb-dev \ - libluajit-5.1-dev \ - libnghttp2-dev \ - libre2-dev \ - libsnmp-dev \ - libsodium-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - make \ - pkg-config \ - ragel \ - systemd \ - python3-venv - - install-clang-8 - - restore-cache-ccache: - product: dnsdist - - get-workspace - - run: - name: autoconf - command: BUILDER_VERSION=0.0.0-git1 autoreconf -vfi - working_directory: ~/project/pdns/dnsdistdist - - run: - name: configure - command: | - CC='clang-8' \ - CXX='clang++-8' \ - CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \ - CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ - ./configure \ - --enable-option-checking=fatal \ - --enable-unit-tests \ - --enable-dnstap \ - --enable-dnscrypt \ - --enable-dns-over-tls \ - --enable-dns-over-https \ - --enable-systemd \ - --prefix=/opt/dnsdist \ - --with-gnutls \ - --with-libsodium \ - --with-lua=luajit \ - --with-libcap \ - --with-nghttp2 \ - --with-re2 \ - --enable-asan \ - --enable-ubsan - working_directory: ~/project/pdns/dnsdistdist - - run: - name: build - command: make -j1 -k - working_directory: ~/project/pdns/dnsdistdist - - save-ccache-cache: - product: dnsdist - - run: - name: Run unit tests - command: make check || (cat test-suite.log; false) - working_directory: ~/project/pdns/dnsdistdist - - run: - name: Install resulting binaries - command: make install - working_directory: ~/project/pdns/dnsdistdist - - persist_to_workspace: - root: /opt - paths: - - dnsdist - - test-ixfrdist-regression: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - install-ixfrdist-deps - - get-workspace - - run: - name: Run ixfrdist tests - workdir: ~/project/regression-tests.ixfrdist - command: | - IXFRDISTBIN="/opt/pdns-auth/bin/ixfrdist" \ - ./runtests - workflows: version: 2 coverity: @@ -1615,9 +1448,6 @@ workflows: - build-auth: requires: - checkout - - test-auth-algorithms: - requires: - - build-auth - test-auth-regress-odbc-sqlite3: requires: - build-auth @@ -1630,9 +1460,6 @@ workflows: - test-auth-regress-ldap: requires: - build-auth - - test-auth-py: - requires: - - build-auth - check-formatting: requires: - checkout @@ -1645,9 +1472,6 @@ workflows: - test-recursor-bulk: requires: - build-recursor - - test-ixfrdist-regression: - requires: - - build-auth build-docs: jobs: diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml index 6173f04191..2d6db51e01 100644 --- a/.github/workflows/build-and-test-all.yml +++ b/.github/workflows/build-and-test-all.yml @@ -201,6 +201,8 @@ jobs: image: mariadb:10 - backend: gpgsql image: postgres:9 + - backend: gsqlite3 # this also runs regression-tests.nobackend and pdnsutil test-algorithms + image: coscale/docker-sleep - backend: lmdb image: coscale/docker-sleep - backend: bind @@ -211,6 +213,8 @@ jobs: image: coscale/docker-sleep - backend: tinydns image: coscale/docker-sleep + - backend: authpy + image: coscale/docker-sleep fail-fast: false services: database: @@ -261,7 +265,7 @@ jobs: path: /opt/pdns-auth - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - run: inv install-clang-runtime - - run: inv install-auth-test-deps -b ${{ matrix.backend }} + - run: inv install-auth-test-deps - run: inv test-ixfrdist test-recursor-api: diff --git a/regression-tests.nobackend/runtests b/regression-tests.nobackend/runtests index fb6cd9cb45..ef50aae6cd 100755 --- a/regression-tests.nobackend/runtests +++ b/regression-tests.nobackend/runtests @@ -1,6 +1,7 @@ #!/bin/sh set -ex cp -f ../regression-tests/zones/test.dyndns.orig ../regression-tests/zones/test.dyndns +cp -f ../regression-tests/zones/sub.test.dyndns.orig ../regression-tests/zones/sub.test.dyndns export testsdir=$(pwd) ../regression-tests/runtests $1 ../regression-tests/toxml @@ -11,7 +12,7 @@ then exit 0; else for t in `cat failed_tests`; do - cat $t/diff + cat $t/diff || true done exit 1; diff --git a/tasks.py b/tasks.py index cda5188d8d..0f2671e130 100644 --- a/tasks.py +++ b/tasks.py @@ -70,6 +70,7 @@ auth_test_deps = [ # FIXME: we should be generating some of these from shlibde 'curl', 'default-jre-headless', 'dnsutils', + 'faketime', 'gawk', 'ldnsutils', 'libboost-serialization1.71.0', @@ -90,7 +91,6 @@ auth_test_deps = [ # FIXME: we should be generating some of these from shlibde 'libsystemd0', 'libyaml-cpp0.6', 'libzmq3-dev', - 'pdns-recursor', 'ruby-bundler', 'ruby-dev', 'socat', @@ -134,7 +134,8 @@ auth_backend_test_deps = dict( bind=[], geoip=[], lua2=[], - tinydns=[] + tinydns=[], + authpy=[] ) @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend']) @@ -151,8 +152,8 @@ def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more tha # rm jdnssec-tools-0.14.tar.gz # fi # echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''') # FIXME: why did this fail with no error? - c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing') # FIXME: can this go? - # FIXME we need to start a background recursor here for some tests + c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing regression-tests.nobackend/rectify-axfr/allow-missing') # FIXME: can this go? + # FIXME we may want to start a background recursor here to make ALIAS tests more robust setup_authbind(c) @task @@ -303,8 +304,8 @@ def ci_auth_install_remotebackend_ruby_deps(c): def ci_auth_run_unit_tests(c): res = c.run('make check', warn=True) if res.exited != 0: - c.run('cat pdns/test-suite.log') - c.run('cat modules/remotebackend/test-suite.log') + c.run('cat pdns/test-suite.log', warn=True) + c.run('cat modules/remotebackend/test-suite.log', warn=True) raise UnexpectedExit(res) @task @@ -401,12 +402,23 @@ def test_auth_backend(c, backend): if backend == 'remote': ci_auth_install_remotebackend_ruby_deps(c) + if backend == 'authpy': + with c.cd('regression-tests.auth-py'): + c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests') + return + with c.cd('regression-tests'): if backend == 'lua2': c.run('touch trustedkeys') # avoid silly error during cleanup for variant in backend_regress_tests[backend]: # FIXME this long line is terrible - c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./start-test-stop 5300 {variant}') + c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./start-test-stop 5300 {variant}') + + if backend == 'gsqlite3': + with c.cd('regression-tests.nobackend'): + c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests') + c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms') + return @task def test_ixfrdist(c):