build(deps): bump actions/download-artifact from 2 to 3

[thirdparty/pdns.git] / .github / workflows / build-and-test-all.yml

---
name: 'Build and test everything'

on:
  push:
  pull_request:
  schedule:
    - cron: '0 22 * * 3'

jobs:
  build-auth:
    name: build auth
    runs-on: ubuntu-20.04
    env:
      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp"
      ASAN_OPTIONS: detect_leaks=0
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: get timestamp for cache
        id: get-stamp
        run: |
          echo "::set-output name=stamp::$(/bin/date +%s)"
        shell: bash
      - name: let GitHub cache our ccache data
        uses: actions/cache@v2
        with:
          path: ~/.ccache
          key: auth-ccache-${{ steps.get-stamp.outputs.stamp }}
          restore-keys: auth-ccache-
      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv install-clang
      - run: inv install-auth-build-deps
      - run: inv ci-autoconf
      - run: inv ci-auth-configure
      - run: inv ci-auth-make
      - run: inv ci-auth-install-remotebackend-test-deps
      - run: inv ci-auth-run-unit-tests
      - run: inv ci-make-install
      - run: ccache -s
      - name: Store the binaries
        uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar
        with:
          name: pdns-auth
          path: /opt/pdns-auth
          retention-days: 1

  build-recursor:
    name: build recursor
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        sanitizers: [ubsan+asan, tsan]
    env:
      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp"
      ASAN_OPTIONS: detect_leaks=0
      SANITIZERS: ${{ matrix.sanitizers }}
    defaults:
      run:
        working-directory: ./pdns/recursordist/
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: get timestamp for cache
        id: get-stamp
        run: |
          echo "::set-output name=stamp::$(/bin/date +%s)"
        shell: bash
      - name: let GitHub cache our ccache data
        uses: actions/cache@v2
        with:
          path: ~/.ccache
          key: recursor-${{ matrix.sanitizers }}-ccache-${{ steps.get-stamp.outputs.stamp }}
          restore-keys: recursor-${{ matrix.sanitizers }}-ccache-
      - run: ../../build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv apt-fresh
      - run: inv install-clang
      - run: inv install-rec-build-deps
      - run: inv ci-autoconf
      - run: inv ci-rec-configure
      - run: inv ci-rec-make
      - run: inv ci-rec-run-unit-tests
      - run: inv ci-make-install
      - run: ccache -s
      - name: Store the binaries
        uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar
        with:
          name: pdns-recursor-${{ matrix.sanitizers }}
          path: /opt/pdns-recursor
          retention-days: 1

  build-dnsdist:
    name: build dnsdist
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        sanitizers: [ubsan+asan, tsan]
        features: [least, full]
        exclude:
          - sanitizers: tsan
            features: least
    env:
      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp"
      ASAN_OPTIONS: detect_leaks=0
      SANITIZERS: ${{ matrix.sanitizers }}
    defaults:
      run:
        working-directory: ./pdns/dnsdistdist/
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: get timestamp for cache
        id: get-stamp
        run: |
          echo "::set-output name=stamp::$(/bin/date +%s)"
        shell: bash
      - name: let GitHub cache our ccache data
        uses: actions/cache@v2
        with:
          path: ~/.ccache
          key: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-ccache-${{ steps.get-stamp.outputs.stamp }}
          restore-keys: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-ccache-
      - run: ../../build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv apt-fresh
      - run: inv install-clang
      - run: inv install-dnsdist-build-deps
      - run: inv ci-autoconf
      - run: inv ci-dnsdist-configure ${{ matrix.features }}
      - run: inv ci-dnsdist-make
      - run: inv ci-dnsdist-run-unit-tests
      - run: inv ci-make-install
      - run: ccache -s
      - name: Store the binaries
        uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar
        with:
          name: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}
          path: /opt/dnsdist
          retention-days: 1

  test-auth-api:
    needs: build-auth
    runs-on: ubuntu-20.04
    env:
      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp"
      ASAN_OPTIONS: detect_leaks=0
      TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/dnsdistdist/dnsdist-tsan.supp"
    strategy:
      matrix:
        include:
          - backend: gsqlite3
            image: coscale/docker-sleep
          - backend: gmysql
            image: mysql:5
          - backend: gpgsql
            image: postgres:9
          - backend: lmdb
            image: coscale/docker-sleep
      fail-fast: false
    services:
      database:
        image: ${{ matrix.image }}
        env:
          POSTGRES_USER: runner
          POSTGRES_HOST_AUTH_METHOD: trust
          MYSQL_ALLOW_EMPTY_PASSWORD: 1
        ports:
          - 3306:3306
          - 5432:5432
        # FIXME: this works around dist-upgrade stopping all docker containers. dist-upgrade is huge on these images anyway. Perhaps we do want to run our tasks in a Docker container too.
        options: >-
          --restart always
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: Fetch the binaries
        uses: actions/download-artifact@v3
        with:
          name: pdns-auth
          path: /opt/pdns-auth
      # - name: Setup upterm session
      #   uses: lhotari/action-upterm@v1
      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv install-clang-runtime
      - run: inv install-auth-test-deps -b ${{ matrix.backend }}
      - run: inv test-api auth -b ${{ matrix.backend }}

  test-auth-backend:
    needs: build-auth
    runs-on: ubuntu-20.04
    env:
      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp"
      ASAN_OPTIONS: detect_leaks=0
    strategy:
      matrix:
        include:
          - backend: remote
            image: coscale/docker-sleep
          - backend: gmysql
            image: mysql:5
          - backend: gmysql
            image: mariadb:10
          - backend: gpgsql
            image: postgres:9
          - backend: gsqlite3  # this also runs regression-tests.nobackend and pdnsutil test-algorithms
            image: coscale/docker-sleep
          - backend: lmdb
            image: coscale/docker-sleep
          - backend: bind
            image: coscale/docker-sleep
          - backend: geoip
            image: coscale/docker-sleep
          - backend: lua2
            image: coscale/docker-sleep
          - backend: tinydns
            image: coscale/docker-sleep
          - backend: authpy
            image: coscale/docker-sleep
      fail-fast: false
    services:
      database:
        image: ${{ matrix.image }}
        env:
          POSTGRES_USER: runner
          POSTGRES_HOST_AUTH_METHOD: trust
          MYSQL_ALLOW_EMPTY_PASSWORD: 1
        ports:
          - 3306:3306
          - 5432:5432
        # FIXME: this works around dist-upgrade stopping all docker containers. dist-upgrade is huge on these images anyway. Perhaps we do want to run our tasks in a Docker container too.
        options: >-
          --restart always
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: Fetch the binaries
        uses: actions/download-artifact@v3
        with:
          name: pdns-auth
          path: /opt/pdns-auth
      # - name: Setup upterm session
      #   uses: lhotari/action-upterm@v1
      # FIXME: install recursor for backends that have ALIAS
      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv install-clang-runtime
      - run: inv install-auth-test-deps -b ${{ matrix.backend }}
      - run: inv test-auth-backend -b ${{ matrix.backend }}

  test-ixfrdist:
    needs: build-auth
    runs-on: ubuntu-20.04
    env:
      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp"
      ASAN_OPTIONS: detect_leaks=0
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: Fetch the binaries
        uses: actions/download-artifact@v3
        with:
          name: pdns-auth
          path: /opt/pdns-auth
      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv install-clang-runtime
      - run: inv install-auth-test-deps
      - run: inv test-ixfrdist

  test-recursor-api:
    needs: build-recursor
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        sanitizers: [ubsan+asan, tsan]
    env:
      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp"
      ASAN_OPTIONS: detect_leaks=0
      TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp"
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: Fetch the binaries
        uses: actions/download-artifact@v3
        with:
          name: pdns-recursor-${{ matrix.sanitizers }}
          path: /opt/pdns-recursor
      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv add-auth-repo  # FIXME: do we need this for rec API testing?
      - run: inv install-clang-runtime
      - run: inv install-rec-test-deps
      - run: inv test-api recursor

  test-recursor-regression:
    needs: build-recursor
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        sanitizers: [ubsan+asan, tsan]
    env:
      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp'
      ASAN_OPTIONS: detect_leaks=0
      TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp"
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: Fetch the binaries
        uses: actions/download-artifact@v3
        with:
          name: pdns-recursor-${{ matrix.sanitizers }}
          path: /opt/pdns-recursor
      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv add-auth-repo
      - run: inv install-clang-runtime
      - run: inv install-rec-test-deps
      - run: inv test-regression-recursor

  test-recursor-bulk:
    name: 'test rec *mini* bulk'
    needs: build-recursor
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        sanitizers: [ubsan+asan]
        threads: [1, 2, 3, 4, 8, 16]
        mthreads: [2048, 4096]
        shards: [1, 2, 1024]
    env:
      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp'
      ASAN_OPTIONS: detect_leaks=0
      TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp"
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: Fetch the binaries
        uses: actions/download-artifact@v3
        with:
          name: pdns-recursor-${{ matrix.sanitizers }}
          path: /opt/pdns-recursor
      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv install-clang-runtime
      - run: inv install-rec-bulk-deps
      - run: inv test-bulk-recursor ${{ matrix.threads }} ${{ matrix.mthreads }} ${{ matrix.shards }}

  test-dnsdist-regression:
    needs: build-dnsdist
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        sanitizers: [ubsan+asan, tsan]
    env:
      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp"
      ASAN_OPTIONS: detect_leaks=0
      TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/dnsdistdist/dnsdist-tsan.supp"
      # IncludeDir tests are disabled because of a weird interaction between TSAN and these tests which ever only happens on GH actions
      SKIP_INCLUDEDIR_TESTS: yes
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: Fetch the binaries
        uses: actions/download-artifact@v3
        with:
          name: dnsdist-full-${{ matrix.sanitizers }}
          path: /opt/dnsdist
      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv install-clang-runtime
      - run: inv install-dnsdist-test-deps
      - run: inv test-dnsdist

  swagger-syntax-check:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
      - run: inv install-swagger-tools
      - run: inv swagger-syntax-check

  collect:
    needs:
      - build-auth
      - build-dnsdist
      - build-recursor
      - swagger-syntax-check
      - test-auth-api
      - test-auth-backend
      - test-dnsdist-regression
      - test-ixfrdist
      - test-recursor-api
      - test-recursor-regression
      - test-recursor-bulk
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 5
          submodules: recursive
      - name: Install yq
        run: sudo wget https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
      - name: Get full list of jobs for this workflow
        run: yq e '.jobs | keys' .github/workflows/build-and-test-all.yml | grep -v '^- collect' | sort | tee /tmp/workflow-jobs-list.yml
      - name: Get list of jobs the collect job depends on
        run: yq e '.jobs.collect.needs | ... comments=""' .github/workflows/build-and-test-all.yml | sort | tee /tmp/workflow-collect-dependencies.yml
      - name: Diff them
        run: diff -u /tmp/workflow-jobs-list.yml /tmp/workflow-collect-dependencies.yml

# FIXME: if we can make upload/download-artifact fasts, running unit tests outside of build can let regression tests start earlier