]> git.ipfire.org Git - thirdparty/pdns.git/commitdiff
gh actions: move testing of almost all auth backends from CircleCI 10711/head
authorPeter van Dijk <peter.van.dijk@powerdns.com>
Fri, 10 Sep 2021 10:14:11 +0000 (12:14 +0200)
committerPeter van Dijk <peter.van.dijk@powerdns.com>
Wed, 22 Sep 2021 08:06:02 +0000 (10:06 +0200)
.circleci/config.yml
.github/workflows/build-and-test-all.yml
tasks.py

index 891ba19c54cfac779d21d4776e64c740521c94ff..71acd24a90fc41a52641c9799b97feb1e732cd60 100644 (file)
@@ -728,30 +728,6 @@ jobs:
           context: godbc_mssql-nsec3-narrow
           skip: 8bit-txt-unescaped
 
-  test-auth-regress-gsqlite3:
-    resource_class: small
-
-    docker:
-      - image: debian:buster
-        auth:
-          username: powerdnsreadonly
-          password: $DOCKERHUB_PASSWORD
-        environment:
-          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
-          ASAN_OPTIONS: detect_leaks=0
-    steps:
-      - auth-regress-setup
-      - auth-regress:
-          context: gsqlite3-nodnssec-both
-      - auth-regress:
-          context: gsqlite3-both
-      - auth-regress:
-          context: gsqlite3-nsec3-both
-      - auth-regress:
-          context: gsqlite3-nsec3-optout-both
-      - auth-regress:
-          context: gsqlite3-nsec3-narrow
-
   test-auth-regress-bind:
     resource_class: small
 
@@ -773,16 +749,6 @@ jobs:
       - auth-regress-setup
       - run:
           command: apt-get install -qq -y p11-kit softhsm2
-      - auth-regress:
-          context: bind-both
-      - auth-regress:
-          context: bind-dnssec-both
-      - auth-regress:
-          context: bind-dnssec-nsec3-both
-      - auth-regress:
-          context: bind-dnssec-nsec3-optout-both
-      - auth-regress:
-          context: bind-dnssec-nsec3-narrow
       - auth-regress:
           context: bind-dnssec-pkcs11
       - run:
@@ -800,94 +766,6 @@ jobs:
       - auth-regress:
           context: bind-hybrid-nsec3
 
-  test-auth-regress-gmysql:
-    resource_class: small
-
-    docker:
-      - image: debian:buster
-        auth:
-          username: powerdnsreadonly
-          password: $DOCKERHUB_PASSWORD
-        environment:
-          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
-          ASAN_OPTIONS: detect_leaks=0
-      - image: mysql:5
-        environment:
-          MYSQL_ALLOW_EMPTY_PASSWORD: 1
-        auth:
-          username: powerdnsreadonly
-          password: $DOCKERHUB_PASSWORD
-    steps:
-      - auth-regress-setup
-      - run:
-          command: apt-get install -qq -y default-mysql-client
-      - run:
-          command: |
-            cat >> ~/.my.cnf \<<- __EOF__
-            [client]
-            protocol=TCP
-            default-character-set=latin1
-            __EOF__
-      - run:
-          name: Set up mysql client lib to force TCP
-          command: echo 'export GMYSQLHOST=127.0.0.1 export GMYSQL2HOST=127.0.0.1 ' > ./vars
-          workdir: ~/project/regression-tests
-      - auth-regress:
-          context: gmysql-nodnssec-both
-      - auth-regress:
-          context: gmysql-both
-      - auth-regress:
-          context: gmysql-nsec3-both
-      - auth-regress:
-          context: gmysql-nsec3-optout-both
-      - auth-regress:
-          context: gmysql-nsec3-narrow
-      - auth-regress:
-          context: gmysql_sp-both
-
-  test-auth-regress-gpgsql:
-    resource_class: small
-
-    docker:
-      - image: debian:buster
-        auth:
-          username: powerdnsreadonly
-          password: $DOCKERHUB_PASSWORD
-        environment:
-          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
-          ASAN_OPTIONS: detect_leaks=0
-      - image: postgres:9
-        auth:
-          username: powerdnsreadonly
-          password: $DOCKERHUB_PASSWORD
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_HOST_AUTH_METHOD: trust
-    steps:
-      - auth-regress-setup
-      - run:
-          command: apt-get install -qq -y postgresql-client
-      - run:
-          name: Use TCP for Postgresql connections
-          command: |
-            cat >> ./vars \<<- __EOF__
-            export PGHOST=127.0.0.1
-            export PGPORT=5432
-            __EOF__
-          workdir: ~/project/regression-tests
-      - auth-regress:
-          context: gpgsql-nodnssec-both
-      - auth-regress:
-          context: gpgsql-both
-      - auth-regress:
-          context: gpgsql-nsec3-both
-      - auth-regress:
-          context: gpgsql-nsec3-optout-both
-      - auth-regress:
-          context: gpgsql-nsec3-narrow
-      - auth-regress:
-          context: gpgsql_sp-both
-
   test-auth-regress-ldap:
     resource_class: small
 
@@ -921,23 +799,6 @@ jobs:
           context: ldap-strict
           doroot: false
 
-  test-auth-regress-tinydns:
-    resource_class: small
-
-    docker:
-      - image: debian:buster
-        auth:
-          username: powerdnsreadonly
-          password: $DOCKERHUB_PASSWORD
-        environment:
-          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
-          ASAN_OPTIONS: detect_leaks=0
-    steps:
-      - auth-regress-setup
-      - auth-regress:
-          context: tinydns
-          doroot: false
-
   test-auth-regress-geoip:
     resource_class: small
 
@@ -951,66 +812,11 @@ jobs:
           ASAN_OPTIONS: detect_leaks=0
     steps:
       - auth-regress-setup
-      - auth-regress:
-          context: geoip
-          doroot: false
-      - auth-regress:
-          context: geoip-nsec3-narrow
-          doroot: false
       - run: export geoipdatabase=../modules/geoipbackend/regression-tests/GeoLiteCity.mmdb
       - auth-regress:
           context: geoip
           doroot: false
 
-  test-auth-regress-lua2:
-    resource_class: small
-
-    docker:
-      - image: debian:buster
-        auth:
-          username: powerdnsreadonly
-          password: $DOCKERHUB_PASSWORD
-        environment:
-          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
-          ASAN_OPTIONS: detect_leaks=0
-    steps:
-      - auth-regress-setup
-      - run:
-          name: Make empty trustedkeys file to avoid useless test script failure
-          workdir: ~/project/regression-tests
-          command: touch trustedkeys
-      - auth-regress:
-          context: lua2
-          doroot: false
-      - auth-regress:
-          context: lua2-dnssec
-          doroot: false
-
-  test-auth-regress-lmdb:
-    resource_class: small
-
-    docker:
-      - image: debian:buster
-        auth:
-          username: powerdnsreadonly
-          password: $DOCKERHUB_PASSWORD
-        environment:
-          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
-          ASAN_OPTIONS: detect_leaks=0
-    steps:
-      - auth-regress-setup
-      - run: apt-get install -y -qq jq
-      - auth-regress:
-          context: lmdb-nodnssec-both
-      - auth-regress:
-          context: lmdb-both
-      - auth-regress:
-          context: lmdb-nsec3-both
-      - auth-regress:
-          context: lmdb-nsec3-optout-both
-      - auth-regress:
-          context: lmdb-nsec3-narrow
-
   test-auth-algorithms:
     resource_class: small
 
@@ -1818,33 +1624,12 @@ workflows:
       - test-auth-regress-odbc-mssql:
           requires:
             - build-auth
-      - test-auth-regress-gmysql:
-          requires:
-            - build-auth
-      - test-auth-regress-bind:
-          requires:
-            - build-auth
       - test-auth-regress-geoip:
           requires:
             - build-auth
-      - test-auth-regress-lua2:
-          requires:
-            - build-auth
-      - test-auth-regress-gsqlite3:
-          requires:
-            - build-auth
-      - test-auth-regress-gpgsql:
-          requires:
-            - build-auth
-      - test-auth-regress-lmdb:
-          requires:
-            - build-auth
       - test-auth-regress-ldap:
           requires:
             - build-auth
-      - test-auth-regress-tinydns:
-          requires:
-            - build-auth
       - test-auth-py:
           requires:
             - build-auth
index 5a4e365f38b1cf4d737db5315c94a2f43eee9f27..6173f04191cfdca8a16a6570eb24b1da31db7e54 100644 (file)
@@ -36,7 +36,6 @@ jobs:
       - run: inv ci-autoconf
       - run: inv ci-auth-configure
       - run: inv ci-auth-make
-      # FIXME: save ccache here?
       - run: inv ci-auth-install-remotebackend-ruby-deps
       - run: inv ci-auth-run-unit-tests
       - run: inv ci-make-install
@@ -80,7 +79,6 @@ jobs:
       - run: inv ci-autoconf
       - run: inv ci-rec-configure
       - run: inv ci-rec-make
-      # FIXME: save ccache here?
       - run: inv ci-rec-run-unit-tests
       - run: inv ci-make-install
       - run: ccache -s
@@ -127,7 +125,6 @@ jobs:
       - run: inv ci-autoconf
       - run: inv ci-dnsdist-configure
       - run: inv ci-dnsdist-make
-      # FIXME: save ccache here?
       - run: inv ci-dnsdist-run-unit-tests
       - run: inv ci-make-install
       - run: ccache -s
@@ -198,12 +195,22 @@ jobs:
         include:
           - backend: remote
             image: coscale/docker-sleep
-          # - backend: gmysql
-          #   image: mysql:5
-          # - backend: gpgsql
-          #   image: postgres:9
-          # - backend: lmdb
-          #   image: coscale/docker-sleep
+          - backend: gmysql
+            image: mysql:5
+          - backend: gmysql
+            image: mariadb:10
+          - backend: gpgsql
+            image: postgres:9
+          - backend: lmdb
+            image: coscale/docker-sleep
+          - backend: bind
+            image: coscale/docker-sleep
+          - backend: geoip
+            image: coscale/docker-sleep
+          - backend: lua2
+            image: coscale/docker-sleep
+          - backend: tinydns
+            image: coscale/docker-sleep
       fail-fast: false
     services:
       database:
@@ -236,6 +243,27 @@ jobs:
       - run: inv install-auth-test-deps -b ${{ matrix.backend }}
       - run: inv test-auth-backend -b ${{ matrix.backend }}
 
+  test-ixfrdist:
+    needs: build-auth
+    runs-on: ubuntu-20.04
+    env:
+      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp'
+      ASAN_OPTIONS: detect_leaks=0
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 5
+          submodules: recursive
+      - name: Fetch the binaries
+        uses: actions/download-artifact@v2
+        with:
+          name: pdns-auth
+          path: /opt/pdns-auth
+      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
+      - run: inv install-clang-runtime
+      - run: inv install-auth-test-deps -b ${{ matrix.backend }}
+      - run: inv test-ixfrdist
+
   test-recursor-api:
     needs: build-recursor
     runs-on: ubuntu-20.04
@@ -286,12 +314,13 @@ jobs:
   collect:
     needs:
       - build-auth
-      - build-recursor
       - build-dnsdist
+      - build-recursor
       - test-auth-api
       - test-auth-backend
-      - test-recursor-api
       - test-dnsdist-regression
+      - test-ixfrdist
+      - test-recursor-api
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2.3.4
index 47ba8bc36a5b77a14ebe47f8dca1945ca62f62ce..ec38456ea409f232dc9ebe84bc011a0c35135095 100644 (file)
--- a/tasks.py
+++ b/tasks.py
@@ -130,7 +130,11 @@ auth_backend_test_deps = dict(
     gmysql=['default-libmysqlclient-dev'],
     gpgsql=['libpq-dev'],
     lmdb=[],
-    remote=[]
+    remote=[],
+    bind=[],
+    geoip=[],
+    lua2=[],
+    tinydns=[]
 )
 
 @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend'])
@@ -350,7 +354,46 @@ def test_api(c, product, backend=''):
         raise Failure('unknown product')
 
 backend_regress_tests = dict(
-    remote = ['pipe', 'unix', 'http', 'zeromq', 'pipe-dnssec', 'unix-dnssec', 'http-dnssec', 'zeromq-dnssec']
+    bind = [
+      'bind-both',
+      'bind-dnssec-both',
+      'bind-dnssec-nsec3-both',
+      'bind-dnssec-nsec3-optout-both',
+      'bind-dnssec-nsec3-narrow',
+    # FIXME  'bind-dnssec-pkcs11'
+    ],
+    geoip = [
+      'geoip',
+      'geoip-nsec3-narrow'
+      # FIXME: also run this with the mmdb we ship
+    ],
+    lua2 = [
+      'lua2',
+      'lua2-dnssec'
+    ],
+    tinydns = [
+      'tinydns'
+    ],
+    remote = [
+      'remotebackend-pipe',
+      'remotebackend-unix',
+      'remotebackend-http',
+      'remotebackend-zeromq',
+      'remotebackend-pipe-dnssec',
+      'remotebackend-unix-dnssec',
+      'remotebackend-http-dnssec',
+      'remotebackend-zeromq-dnssec'
+    ],
+    lmdb = [
+      'lmdb-nodnssec-both',
+      'lmdb-both',
+      'lmdb-nsec3-both',
+      'lmdb-nsec3-optout-both',
+      'lmdb-nsec3-narrow'
+    ],
+    gmysql   = ['gmysql',     'gmysql-nodnssec-both',   'gmysql-nsec3-both',   'gmysql-nsec3-optout-both',   'gmysql-nsec3-narrow',   'gmysql_sp-both'],
+    gpgsql   = ['gpgsql',     'gpgsql-nodnssec-both',   'gpgsql-nsec3-both',   'gpgsql-nsec3-optout-both',   'gpgsql-nsec3-narrow',   'gpgsql_sp-both'],
+    gsqlite3 = ['gsqlite3', 'gsqlite3-nodnssec-both', 'gsqlite3-nsec3-both', 'gsqlite3-nsec3-optout-both', 'gsqlite3-nsec3-narrow'],
 )
 
 @task
@@ -359,10 +402,16 @@ def test_auth_backend(c, backend):
         ci_auth_install_remotebackend_ruby_deps(c)
 
     with c.cd('regression-tests'):
-        for t in backend_regress_tests[backend]:
+        if backend == 'lua2':
+            c.run('touch trustedkeys')  # avoid silly error during cleanup
+        for variant in backend_regress_tests[backend]:
             # FIXME this long line is terrible
-            # FIXME this appends 'backend' but that's only correct for 'remote'
-            c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./start-test-stop 5300 {backend}backend-{t}')
+            c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./start-test-stop 5300 {variant}')
+
+@task
+def test_ixfrdist(c):
+    with c.cd('regression-tests.ixfrdist'):
+        c.run('IXFRDISTBIN=/opt/pdns-auth/bin/ixfrdist ./runtests')
 
 @task
 def test_dnsdist(c):