From 7d862cb38635506af5faaa2b0bae7ab6189e092e Mon Sep 17 00:00:00 2001 From: Alexis Romero Date: Fri, 12 May 2023 11:30:50 +0200 Subject: [PATCH] Run build-and-test-all.yml on a custom debian container --- .github/workflows/build-and-test-all.yml | 188 +++++++++--------- .../kerberos-client/krb5.conf | 4 +- regression-tests.dnsdist/dnsdisttests.py | 2 +- regression-tests.dnsdist/test_Advanced.py | 3 + regression-tests.nobackend/counters/command | 18 +- .../counters/expected_result.noipv6 | 71 +++++++ .../distributor/command | 8 +- tasks.py | 76 +++---- 8 files changed, 236 insertions(+), 134 deletions(-) create mode 100644 regression-tests.nobackend/counters/expected_result.noipv6 diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml index 94e8db2e09..3e0c3fe1be 100644 --- a/.github/workflows/build-and-test-all.yml +++ b/.github/workflows/build-and-test-all.yml @@ -10,21 +10,28 @@ on: permissions: # least privileges, see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions contents: read +env: + CLANG_VERSION: '13' + # github.workspace variable points to the Runner home folder. Container home folder defined below. + REPO_HOME: '/__w/pdns/pdns' + jobs: build-auth: name: build auth if: ${{ !github.event.schedule || vars.SCHEDULED_JOBS_BUILD_AND_TEST_ALL }} runs-on: ubuntu-20.04 - env: - ASAN_OPTIONS: detect_leaks=0 - FUZZING_TARGETS: yes - SANITIZERS: asan+ubsan - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" - UNIT_TESTS: yes + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + ASAN_OPTIONS: detect_leaks=0 + FUZZING_TARGETS: yes + SANITIZERS: asan+ubsan + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp" + UNIT_TESTS: yes + PDNS_TEST_NO_IPV6: yes outputs: clang-tidy-failed: ${{ steps.clang-tidy-annotations.outputs.failed }} steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 @@ -34,23 +41,20 @@ jobs: run: | echo "stamp=$(/bin/date +%s)" >> "$GITHUB_OUTPUT" shell: bash + - run: mkdir -p ~/.ccache - name: let GitHub cache our ccache data uses: actions/cache@v3 with: path: ~/.ccache key: auth-ccache-${{ steps.get-stamp.outputs.stamp }} restore-keys: auth-ccache- - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - - run: inv install-clang - - run: inv install-clang-tidy-tools - - run: inv install-auth-build-deps - run: inv ci-autoconf - run: inv ci-auth-configure - run: inv ci-auth-make-bear # This runs under pdns/ - run: ln -s .clang-tidy.full .clang-tidy - name: Run clang-tidy working-directory: pdns - run: git diff -U0 HEAD^..HEAD | python3 ../.github/scripts/git-filter.py | python3 /usr/bin/clang-tidy-diff-12.py -clang-tidy-binary /usr/bin/clang-tidy-12 -extra-arg=-ferror-limit=0 -p2 -export-fixes clang-tidy-auth.yml + run: git diff -U0 HEAD^..HEAD | python3 ../.github/scripts/git-filter.py | python3 /usr/bin/clang-tidy-diff-${CLANG_VERSION}.py -clang-tidy-binary /usr/bin/clang-tidy-${CLANG_VERSION} -extra-arg=-ferror-limit=0 -p2 -export-fixes clang-tidy-auth.yml - name: Print clang-tidy fixes YAML working-directory: pdns shell: bash @@ -86,18 +90,19 @@ jobs: strategy: matrix: sanitizers: [ubsan+asan, tsan] - env: - ASAN_OPTIONS: detect_leaks=0 - SANITIZERS: ${{ matrix.sanitizers }} - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" - UNIT_TESTS: yes + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + ASAN_OPTIONS: detect_leaks=0 + SANITIZERS: ${{ matrix.sanitizers }} + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp" + UNIT_TESTS: yes defaults: run: working-directory: ./pdns/recursordist/ outputs: clang-tidy-failed: ${{ steps.clang-tidy-annotations.outputs.failed }} steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 @@ -107,23 +112,19 @@ jobs: run: | echo "stamp=$(/bin/date +%s)" >> "$GITHUB_OUTPUT" shell: bash + - run: mkdir -p ~/.ccache - name: let GitHub cache our ccache data uses: actions/cache@v3 with: path: ~/.ccache key: recursor-${{ matrix.sanitizers }}-ccache-${{ steps.get-stamp.outputs.stamp }} restore-keys: recursor-${{ matrix.sanitizers }}-ccache- - - run: ../../build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - - run: inv apt-fresh - - run: inv install-clang - - run: inv install-clang-tidy-tools - - run: inv install-rec-build-deps - run: inv ci-autoconf - run: inv ci-rec-configure - run: inv ci-rec-make-bear - run: ln -s ../../.clang-tidy.full .clang-tidy - name: Run clang-tidy - run: git diff -U0 HEAD^..HEAD | python3 ../../.github/scripts/git-filter.py | python3 /usr/bin/clang-tidy-diff-12.py -clang-tidy-binary /usr/bin/clang-tidy-12 -extra-arg=-ferror-limit=0 -p3 -export-fixes clang-tidy-rec.yml + run: git diff -U0 HEAD^..HEAD | python3 ../../.github/scripts/git-filter.py | python3 /usr/bin/clang-tidy-diff-${CLANG_VERSION}.py -clang-tidy-binary /usr/bin/clang-tidy-${CLANG_VERSION} -extra-arg=-ferror-limit=0 -p3 -export-fixes clang-tidy-rec.yml - name: Print clang-tidy fixes YAML shell: bash run: | @@ -160,18 +161,19 @@ jobs: exclude: - sanitizers: tsan features: least - env: - ASAN_OPTIONS: detect_leaks=0 - SANITIZERS: ${{ matrix.sanitizers }} - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" - UNIT_TESTS: yes + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + ASAN_OPTIONS: detect_leaks=0 + SANITIZERS: ${{ matrix.sanitizers }} + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp" + UNIT_TESTS: yes defaults: run: working-directory: ./pdns/dnsdistdist/ outputs: clang-tidy-failed: ${{ steps.clang-tidy-annotations.outputs.failed }} steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 @@ -181,23 +183,19 @@ jobs: run: | echo "stamp=$(/bin/date +%s)" >> "$GITHUB_OUTPUT" shell: bash + - run: mkdir -p ~/.ccache - name: let GitHub cache our ccache data uses: actions/cache@v3 with: path: ~/.ccache key: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-ccache-${{ steps.get-stamp.outputs.stamp }} restore-keys: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-ccache- - - run: ../../build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - - run: inv apt-fresh - - run: inv install-clang - - run: inv install-clang-tidy-tools - - run: inv install-dnsdist-build-deps - run: inv ci-autoconf - run: inv ci-dnsdist-configure ${{ matrix.features }} - run: inv ci-dnsdist-make-bear - run: ln -s ../../.clang-tidy.full .clang-tidy - name: Run clang-tidy - run: git diff -U0 HEAD^..HEAD | python3 ../../.github/scripts/git-filter.py | python3 /usr/bin/clang-tidy-diff-12.py -clang-tidy-binary /usr/bin/clang-tidy-12 -extra-arg=-ferror-limit=0 -p3 -export-fixes clang-tidy-dnsdist.yml + run: git diff -U0 HEAD^..HEAD | python3 ../../.github/scripts/git-filter.py | python3 /usr/bin/clang-tidy-diff-${CLANG_VERSION}.py -clang-tidy-binary /usr/bin/clang-tidy-${CLANG_VERSION} -extra-arg=-ferror-limit=0 -p3 -export-fixes clang-tidy-dnsdist.yml - name: Print clang-tidy fixes YAML shell: bash run: | @@ -226,10 +224,13 @@ jobs: test-auth-api: needs: build-auth runs-on: ubuntu-20.04 - env: - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" - ASAN_OPTIONS: detect_leaks=0 - TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/dnsdistdist/dnsdist-tsan.supp" + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp" + ASAN_OPTIONS: detect_leaks=0 + TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ env.REPO_HOME }}/pdns/dnsdistdist/dnsdist-tsan.supp" + AUTH_BACKEND_IP_ADDR: "172.17.0.1" strategy: matrix: include: @@ -256,7 +257,6 @@ jobs: options: >- --restart always steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 @@ -266,9 +266,7 @@ jobs: with: name: pdns-auth path: /opt/pdns-auth - # - name: Setup upterm session - # uses: lhotari/action-upterm@v1 - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv apt-fresh - run: inv install-clang-runtime - run: inv install-auth-test-deps -b ${{ matrix.backend }} - run: inv test-api auth -b ${{ matrix.backend }} @@ -276,10 +274,15 @@ jobs: test-auth-backend: needs: build-auth runs-on: ubuntu-20.04 - env: - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" - ASAN_OPTIONS: detect_leaks=0 - LDAPHOST: ldap://ldapserver/ + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp" + ASAN_OPTIONS: detect_leaks=0 + LDAPHOST: ldap://ldapserver/ + ODBCINI: /github/home/.odbc.ini + AUTH_BACKEND_IP_ADDR: "172.17.0.1" + SKIP_IPV6_TESTS: yes strategy: matrix: include: @@ -366,7 +369,6 @@ jobs: options: >- --restart always steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 @@ -376,10 +378,7 @@ jobs: with: name: pdns-auth path: /opt/pdns-auth - # - name: Setup upterm session - # uses: lhotari/action-upterm@v1 # FIXME: install recursor for backends that have ALIAS - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - run: inv install-clang-runtime - run: inv install-auth-test-deps -b ${{ matrix.backend }} - run: inv test-auth-backend -b ${{ matrix.backend }} @@ -387,11 +386,12 @@ jobs: test-ixfrdist: needs: build-auth runs-on: ubuntu-20.04 - env: - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" - ASAN_OPTIONS: detect_leaks=0 + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp" + ASAN_OPTIONS: detect_leaks=0 steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 @@ -401,7 +401,6 @@ jobs: with: name: pdns-auth path: /opt/pdns-auth - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - run: inv install-clang-runtime - run: inv install-auth-test-deps - run: inv test-ixfrdist @@ -412,12 +411,16 @@ jobs: strategy: matrix: sanitizers: [ubsan+asan, tsan] - env: - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" - ASAN_OPTIONS: detect_leaks=0 - TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp" + dist_name: [debian] + dist_release_name: [bullseye] + pdns_repo_version: ['45'] + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp" + ASAN_OPTIONS: detect_leaks=0 + TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ env.REPO_HOME }}/pdns/recursordist/recursor-tsan.supp" steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 @@ -427,8 +430,8 @@ jobs: with: name: pdns-recursor-${{ matrix.sanitizers }} path: /opt/pdns-recursor - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - - run: inv add-auth-repo # FIXME: do we need this for rec API testing? + - run: inv apt-fresh + - run: inv add-auth-repo ${{ matrix.dist_name }} ${{ matrix.dist_release_name }} ${{ matrix.pdns_repo_version }} - run: inv install-clang-runtime - run: inv install-rec-test-deps - run: inv test-api recursor @@ -439,13 +442,19 @@ jobs: strategy: matrix: sanitizers: [ubsan+asan, tsan] - env: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp' - ASAN_OPTIONS: detect_leaks=0 - TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp" + dist_name: [debian] + dist_release_name: [bullseye] + pdns_repo_version: ['45'] + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ env.REPO_HOME }}/pdns/recursordist/recursor-tsan.supp" + SKIP_IPV6_TESTS: yes steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - - uses: actions/checkout@v3 + # - uses: PowerDNS/pdns/set-ubuntu-mirror@meta + - uses: actions/checkout@v3.1.0 with: fetch-depth: 5 submodules: recursive @@ -454,8 +463,8 @@ jobs: with: name: pdns-recursor-${{ matrix.sanitizers }} path: /opt/pdns-recursor - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - - run: inv add-auth-repo + - run: inv apt-fresh + - run: inv add-auth-repo ${{ matrix.dist_name }} ${{ matrix.dist_release_name }} ${{ matrix.pdns_repo_version }} - run: inv install-clang-runtime - run: inv install-rec-test-deps - run: inv test-regression-recursor @@ -470,12 +479,13 @@ jobs: threads: [1, 2, 3, 4, 8] mthreads: [2048] shards: [1, 2, 1024] - env: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp' - ASAN_OPTIONS: detect_leaks=0 - TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp" + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ env.REPO_HOME }}/pdns/recursordist/recursor-tsan.supp" steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 @@ -485,7 +495,6 @@ jobs: with: name: pdns-recursor-${{ matrix.sanitizers }} path: /opt/pdns-recursor - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - run: inv install-clang-runtime - run: inv install-rec-bulk-deps - run: inv test-bulk-recursor ${{ matrix.threads }} ${{ matrix.mthreads }} ${{ matrix.shards }} @@ -496,15 +505,17 @@ jobs: strategy: matrix: sanitizers: [ubsan+asan, tsan] - env: - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" - # Disabling (intercept_send=0) the custom send wrappers for ASAN and TSAN because they cause the tools to report a race that doesn't exist on actual implementations of send(), see https://github.com/google/sanitizers/issues/1498 - ASAN_OPTIONS: detect_leaks=0:intercept_send=0 - TSAN_OPTIONS: "halt_on_error=1:intercept_send=0:suppressions=${{ github.workspace }}/pdns/dnsdistdist/dnsdist-tsan.supp" - # IncludeDir tests are disabled because of a weird interaction between TSAN and these tests which ever only happens on GH actions - SKIP_INCLUDEDIR_TESTS: yes + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master + env: + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ env.REPO_HOME }}/build-scripts/UBSan.supp" + # Disabling (intercept_send=0) the custom send wrappers for ASAN and TSAN because they cause the tools to report a race that doesn't exist on actual implementations of send(), see https://github.com/google/sanitizers/issues/1498 + ASAN_OPTIONS: detect_leaks=0:intercept_send=0 + TSAN_OPTIONS: "halt_on_error=1:intercept_send=0:suppressions=${{ env.REPO_HOME }}/pdns/dnsdistdist/dnsdist-tsan.supp" + # IncludeDir tests are disabled because of a weird interaction between TSAN and these tests which ever only happens on GH actions + SKIP_INCLUDEDIR_TESTS: yes + SKIP_IPV6_TESTS: yes steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 @@ -514,7 +525,6 @@ jobs: with: name: dnsdist-full-${{ matrix.sanitizers }} path: /opt/dnsdist - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - run: inv install-clang-runtime - run: inv install-dnsdist-test-deps - run: inv test-dnsdist @@ -522,13 +532,13 @@ jobs: swagger-syntax-check: if: ${{ !github.event.schedule || vars.SCHEDULED_JOBS_BUILD_AND_TEST_ALL }} runs-on: ubuntu-20.04 + container: + image: ghcr.io/powerdns/base-pdns-ci-image/debian-11-pdns-base:master steps: - - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - uses: actions/checkout@v3 with: fetch-depth: 5 submodules: recursive - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - run: inv install-swagger-tools - run: inv swagger-syntax-check diff --git a/regression-tests.auth-py/kerberos-client/krb5.conf b/regression-tests.auth-py/kerberos-client/krb5.conf index 1d65c48436..287fcc290b 100755 --- a/regression-tests.auth-py/kerberos-client/krb5.conf +++ b/regression-tests.auth-py/kerberos-client/krb5.conf @@ -3,7 +3,7 @@ [realms] EXAMPLE.COM = { - kdc = 127.0.0.1:1188 - admin_server = 127.0.0.1:1749 + kdc = kerberos-server:1188 + admin_server = kerberos-server:1749 } diff --git a/regression-tests.dnsdist/dnsdisttests.py b/regression-tests.dnsdist/dnsdisttests.py index dfc6ead966..ce3e3ccf81 100644 --- a/regression-tests.dnsdist/dnsdisttests.py +++ b/regression-tests.dnsdist/dnsdisttests.py @@ -764,7 +764,7 @@ class DNSDistTest(AssertEqualDNSMessageMixin, unittest.TestCase): return result.decode('UTF-8') @classmethod - def sendConsoleCommand(cls, command, timeout=1.0): + def sendConsoleCommand(cls, command, timeout=5.0): ourNonce = libnacl.utils.rand_nonce() theirNonce = None sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/regression-tests.dnsdist/test_Advanced.py b/regression-tests.dnsdist/test_Advanced.py index 1a110e483d..5c3efbb501 100644 --- a/regression-tests.dnsdist/test_Advanced.py +++ b/regression-tests.dnsdist/test_Advanced.py @@ -394,6 +394,9 @@ class TestAdvancedGetLocalAddressOnAnyBind(DNSDistTest): self.assertEqual(receivedQuery, query) self.assertEqual(receivedResponse, response) + if 'SKIP_IPV6_TESTS' in os.environ: + return + # a bit more tricky, UDP-only IPv6 sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) sock.settimeout(1.0) diff --git a/regression-tests.nobackend/counters/command b/regression-tests.nobackend/counters/command index a88d4a46e4..8647f5d9c3 100755 --- a/regression-tests.nobackend/counters/command +++ b/regression-tests.nobackend/counters/command @@ -4,11 +4,17 @@ if [ "${PDNS_DEBUG}" = "YES" ]; then set -x fi +local_address="127.0.0.1,::1" + +if [ -n "${SKIP_IPV6_TESTS}" ]; then + local_address="127.0.0.1" +fi + port=5600 rm -f pdns*.pid -$PDNS --daemon=no --local-address=127.0.0.1,::1 \ +$PDNS --daemon=no --local-address=$local_address \ --local-port=$port --socket-dir=./ --no-shuffle --launch=bind --no-config \ --module-dir=../regression-tests/modules --bind-config=counters/named.conf & @@ -20,11 +26,13 @@ $SDIG 127.0.0.1 $port test.com SOA >&2 >/dev/null $SDIG 127.0.0.1 $port server1.test.com A tcp >&2 >/dev/null $SDIG 127.0.0.1 $port test.com SOA tcp >&2 >/dev/null -$SDIG ::1 $port server1.test.com A >&2 >/dev/null -$SDIG ::1 $port server1.com A tcp >&2 >/dev/null +if [ -z "${SKIP_IPV6_TESTS}" ]; then + $SDIG ::1 $port server1.test.com A >&2 >/dev/null + $SDIG ::1 $port server1.com A tcp >&2 >/dev/null -$SDIG ::1 $port test.com SOA >&2 >/dev/null -$SDIG ::1 $port test.com SOA tcp >&2 >/dev/null + $SDIG ::1 $port test.com SOA >&2 >/dev/null + $SDIG ::1 $port test.com SOA tcp >&2 >/dev/null +fi # NXDOMAIN $SDIG 127.0.0.1 $port nx.test.com A >&2 >/dev/null diff --git a/regression-tests.nobackend/counters/expected_result.noipv6 b/regression-tests.nobackend/counters/expected_result.noipv6 new file mode 100644 index 0000000000..650b2c96cd --- /dev/null +++ b/regression-tests.nobackend/counters/expected_result.noipv6 @@ -0,0 +1,71 @@ + +corrupt-packets=0 +deferred-cache-inserts=0 +deferred-cache-lookup=0 +deferred-packetcache-inserts=0 +deferred-packetcache-lookup=0 +dnsupdate-answers=0 +dnsupdate-changes=0 +dnsupdate-queries=0 +dnsupdate-refused=0 +incoming-notifications=0 +key-cache-size=0 +meta-cache-size=1 +noerror-packets=1 +nxdomain-packets=1 +open-tcp-connections=0 +overload-drops=0 +packetcache-size=7 +qsize-q=0 +query-cache-size=4 +rd-queries=0 +recursing-answers=0 +recursing-questions=0 +recursion-unanswered=0 +ring-logmessages-capacity=10000 +ring-logmessages-size=0 +ring-noerror-queries-capacity=10000 +ring-noerror-queries-size=0 +ring-nxdomain-queries-capacity=10000 +ring-nxdomain-queries-size=0 +ring-queries-capacity=10000 +ring-queries-size=0 +ring-remotes-capacity=10000 +ring-remotes-corrupt-capacity=10000 +ring-remotes-corrupt-size=0 +ring-remotes-size=0 +ring-remotes-unauth-capacity=10000 +ring-remotes-unauth-size=0 +ring-servfail-queries-capacity=10000 +ring-servfail-queries-size=0 +ring-unauth-queries-capacity=10000 +ring-unauth-queries-size=0 +security-status=0 +servfail-packets=0 +signature-cache-size=0 +signatures=0 +tcp-answers-bytes=128 +tcp-answers=2 +tcp-cookie-queries=0 +tcp-queries=2 +tcp4-answers-bytes=128 +tcp4-answers=2 +tcp4-queries=2 +tcp6-answers-bytes=0 +tcp6-answers=0 +tcp6-queries=0 +timedout-packets=0 +udp-answers-bytes=321 +udp-answers=5 +udp-cookie-queries=0 +udp-do-queries=0 +udp-queries=5 +udp4-answers-bytes=321 +udp4-answers=5 +udp4-queries=5 +udp6-answers-bytes=0 +udp6-answers=0 +udp6-queries=0 +unauth-packets=1 +xfr-queue=0 +zone-cache-size=1 diff --git a/regression-tests.nobackend/distributor/command b/regression-tests.nobackend/distributor/command index c96f3770b6..ec37c64da8 100755 --- a/regression-tests.nobackend/distributor/command +++ b/regression-tests.nobackend/distributor/command @@ -5,11 +5,17 @@ if [ "${PDNS_DEBUG}" = "YES" ]; then set -x fi +local_address="127.0.0.1,::1" + +if [ -n "${SKIP_IPV6_TESTS}" ]; then + local_address="127.0.0.1" +fi + port=5600 rm -f pdns*.pid -$PDNS --daemon=no --local-address=127.0.0.1,::1 \ +$PDNS --daemon=no --local-address=$local_address \ --local-port=$port --socket-dir=./ --no-shuffle --launch=pipe --no-config \ --module-dir=../regression-tests/modules --pipe-command=$(pwd)/distributor/slow.pl \ --pipe-abi-version=5 \ diff --git a/tasks.py b/tasks.py index 33bdfc70a2..426a428f3b 100644 --- a/tasks.py +++ b/tasks.py @@ -5,6 +5,10 @@ import os import sys import time +auth_backend_ip_addr = os.getenv('AUTH_BACKEND_IP_ADDR', '127.0.0.1') + +clang_version = os.getenv('CLANG_VERSION', '13') + all_build_deps = [ 'ccache', 'libboost-all-dev', @@ -59,7 +63,7 @@ rec_bulk_deps = [ 'libcap2', 'libfstrm0', 'libluajit-5.1-2', - 'libsnmp35', + '"libsnmp[1-9]+"', 'libsodium23', 'libssl1.1', 'libsystemd0', @@ -91,7 +95,7 @@ auth_test_deps = [ # FIXME: we should be generating some of these from shlibde 'gawk', 'krb5-user', 'ldnsutils', - 'libboost-serialization1.71.0', + '"libboost-serialization1.7[1-9]+"', 'libcdb1', 'libcurl4', 'libgeoip1', @@ -153,24 +157,24 @@ def apt_fresh(c): @task def install_clang(c): """ - install clang-12 and llvm-12 + install clang and llvm """ - c.sudo('apt-get -y --no-install-recommends install clang-12 llvm-12') + c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version} llvm-{clang_version}') @task def install_clang_tidy_tools(c): - c.sudo('apt-get -y --no-install-recommends install clang-tidy-12 clang-tools-12 bear python-yaml') + c.sudo(f'apt-get -y --no-install-recommends install clang-tidy-{clang_version} clang-tools-{clang_version} bear python3-yaml') @task def install_clang_runtime(c): # this gives us the symbolizer, for symbols in asan/ubsan traces - c.sudo('apt-get -y --no-install-recommends install clang-12') + c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version}') def install_libdecaf(c, product): c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf') with c.cd('/tmp/libdecaf'): c.run('git checkout 41f349') - c.run('CC=clang-12 CXX=clang-12 ' + c.run(f'CC=clang-{clang_version} CXX=clang-{clang_version} ' 'cmake -B build ' '-DCMAKE_INSTALL_PREFIX=/usr/local ' '-DCMAKE_INSTALL_LIBDIR=lib ' @@ -221,7 +225,7 @@ def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more tha extra=[] for b in backend: extra.extend(auth_backend_test_deps[b]) - c.sudo('apt-get -y install ' + ' '.join(extra+auth_test_deps)) + c.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install ' + ' '.join(extra+auth_test_deps)) c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*') # c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then @@ -256,7 +260,7 @@ def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get setup_authbind(c) c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf') - c.sudo('systemctl restart snmpd') + c.sudo('/etc/init.d/snmpd restart') time.sleep(5) c.sudo('chmod 755 /var/agentx') @@ -273,7 +277,7 @@ def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt- libh2o-evloop0.13 \ liblmdb0 \ libnghttp2-14 \ - libre2-5 \ + "libre2-[1-9]+" \ libssl-dev \ libsystemd0 \ libsodium23 \ @@ -282,7 +286,7 @@ def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt- protobuf-compiler \ python3-venv snmpd prometheus') c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf') - c.sudo('systemctl restart snmpd') + c.sudo('/etc/init.d/snmpd restart') time.sleep(5) c.sudo('chmod 755 /var/agentx') @@ -361,8 +365,8 @@ def get_base_configure_cmd(): f'CFLAGS="{get_cflags()}"', f'CXXFLAGS="{get_cxxflags()}"', './configure', - "CC='clang-12'", - "CXX='clang++-12'", + f"CC='clang-{clang_version}'", + f"CXX='clang++-{clang_version}'", "--enable-option-checking=fatal", "--enable-systemd", "--with-libsodium", @@ -509,13 +513,13 @@ def ci_dnsdist_configure(c, features): sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else '' cflags = '-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int' cxxflags = cflags + ' -Wp,-D_GLIBCXX_ASSERTIONS ' + additional_flags - res = c.run('''CFLAGS="%s" \ + res = c.run(f'''CFLAGS="%s" \ CXXFLAGS="%s" \ - AR=llvm-ar-12 \ - RANLIB=llvm-ranlib-12 \ + AR=llvm-ar-{clang_version} \ + RANLIB=llvm-ranlib-{clang_version} \ ./configure \ - CC='clang-12' \ - CXX='clang++-12' \ + CC='clang-{clang_version}' \ + CXX='clang++-{clang_version}' \ --enable-option-checking=fatal \ --enable-fortify-source=auto \ --enable-auto-var-init=pattern \ @@ -533,7 +537,7 @@ def ci_auth_make(c): def ci_auth_make_bear(c): # Needed for clang-tidy -line-filter vs project structure shenanigans with c.cd('pdns'): - c.run('bear --append make -j8 -k V=1 -C ..') + c.run('bear --append -- make -j8 -k V=1 -C ..') @task def ci_rec_make(c): @@ -542,7 +546,7 @@ def ci_rec_make(c): @task def ci_rec_make_bear(c): # Assumed to be running under ./pdns/recursordist/ - c.run('bear --append make -j8 -k V=1') + c.run('bear --append -- make -j8 -k V=1') @task def ci_dnsdist_make(c): @@ -551,7 +555,7 @@ def ci_dnsdist_make(c): @task def ci_dnsdist_make_bear(c): # Assumed to be running under ./pdns/dnsdistdist/ - c.run('bear --append make -j4 -k V=1') + c.run('bear --append -- make -j4 -k V=1') @task def ci_auth_install_remotebackend_test_deps(c): @@ -587,17 +591,13 @@ def ci_make_install(c): res = c.run('make install') # FIXME: this builds auth docs - again @task -def add_auth_repo(c): - dist = 'ubuntu' # FIXME take these from the caller? - release = 'focal' - version = '44' - +def add_auth_repo(c, dist_name, dist_release_name, pdns_repo_version): c.sudo('apt-get install -y curl gnupg2') - if version == 'master': + if pdns_repo_version == 'master': c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc') else: c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc') - c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist} {release}-auth-{version} main' | sudo tee /etc/apt/sources.list.d/pdns.list") + c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist_name} {dist_release_name}-auth-{pdns_repo_version} main' | sudo tee /etc/apt/sources.list.d/pdns.list") c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns") c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns") c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns") @@ -610,7 +610,7 @@ def test_api(c, product, backend=''): c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}') elif product == 'auth': with c.cd('regression-tests.api'): - c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests authoritative {backend}') + c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432 ./runtests authoritative {backend}') else: raise Failure('unknown product') @@ -687,11 +687,11 @@ backend_regress_tests = dict( godbc_mssql_credentials = {"username": "sa", "password": "SAsa12%%"} -godbc_config = ''' +godbc_config = f''' [pdns-mssql-docker] Driver=FreeTDS Trace=No -Server=127.0.0.1 +Server={auth_backend_ip_addr} Port=1433 Database=pdns TDS_Version=7.1 @@ -699,7 +699,7 @@ TDS_Version=7.1 [pdns-mssql-docker-nodb] Driver=FreeTDS Trace=No -Server=127.0.0.1 +Server={auth_backend_ip_addr} Port=1433 TDS_Version=7.1 @@ -728,16 +728,17 @@ def setup_godbc_sqlite3(c): def setup_ldap_client(c): c.sudo('DEBIAN_FRONTEND=noninteractive apt-get install -y ldap-utils') - c.sudo('sh -c \'echo "127.0.0.1 ldapserver" | tee -a /etc/hosts\'') + c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} ldapserver" | tee -a /etc/hosts\'') @task def test_auth_backend(c, backend): - pdns_auth_env_vars = 'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432"' + pdns_auth_env_vars = f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST={auth_backend_ip_addr} GMYSQL2HOST={auth_backend_ip_addr} MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432' if backend == 'remote': ci_auth_install_remotebackend_test_deps(c) if backend == 'authpy': + c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} kerberos-server" | tee -a /etc/hosts\'') with c.cd('regression-tests.auth-py'): c.run(f'{pdns_auth_env_vars} WITHKERBEROS=YES ./runtests') return @@ -772,7 +773,10 @@ def test_auth_backend(c, backend): c.run(f'{pdns_auth_env_vars} ./start-test-stop 5300 {variant}') if backend == 'gsqlite3': + if os.getenv('SKIP_IPV6_TESTS'): + pdns_auth_env_vars += ' context=noipv6' with c.cd('regression-tests.nobackend'): + c.run(f'echo {pdns_auth_env_vars}') c.run(f'{pdns_auth_env_vars} ./runtests') c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms') return @@ -793,7 +797,7 @@ def test_dnsdist(c): @task def test_regression_recursor(c): c.run('/opt/pdns-recursor/sbin/pdns_recursor --version') - c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control SKIP_IPV6_TESTS=y ./build-scripts/test-recursor') + c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control ./build-scripts/test-recursor') @task def test_bulk_recursor(c, threads, mthreads, shards): @@ -819,7 +823,7 @@ def install_coverity_tools(c, project): @task def coverity_clang_configure(c): - c.sudo('/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-12') + c.sudo(f'/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-{clang_version}') @task def coverity_make(c): -- 2.47.2