Merge pull request #12485 from omoerbeek/rec-prep-4.8.2

[thirdparty/pdns.git] / .circleci / config.yml

version: 2.1

commands:
  checkout-shallow:
    description: "Do a shallow checkout of the repository"
    steps:
      - run:
          name: Install git and openssh
          command: apt-get update && apt-get -qq -y install openssh-client git
      - run:
          name: Store the keyid for the git repo
          command: mkdir -p $HOME/.ssh && ssh-keyscan $(echo ${CIRCLE_REPOSITORY_URL} | sed 's,.*@\([^:]*\):.*,\1,') > $HOME/.ssh/known_hosts
      - run:
          name: Clone the repo
          command: |
            mkdir -p /opt/project
            if [ -n "$CIRCLE_PR_NUMBER" ]
            then
              echo === Checking out PR "$CIRCLE_PR_NUMBER" from "$CIRCLE_REPOSITORY_URL"
              git clone --depth 1 $CIRCLE_REPOSITORY_URL /opt/project
              cd /opt/project
              git fetch --depth 1 origin +refs/pull/${CIRCLE_PR_NUMBER}/merge
              git checkout -qf FETCH_HEAD
            else
              echo === Checking out branch "${CIRCLE_BRANCH}" from "$CIRCLE_REPOSITORY_URL"
              git clone --depth 1 --branch $CIRCLE_BRANCH $CIRCLE_REPOSITORY_URL /opt/project
            fi
            cd /opt/project
            git --no-pager show -s

  get-workspace:
    description: "Attach workspace to /opt and symlink checkout into home"
    steps:
      - run:
          name: ensure ca-certificates is installed
          command: apt-get update && apt-get -qq -y install ca-certificates
      - attach_workspace:
          at: /opt
      - run:
          name: symlink checkout into home
          command: rmdir ~/project && ln -s /opt/project ~/project
      - run:
          name: include $BASH_ENV for both interactive and non-interactive shells
          command: echo '. $BASH_ENV' >> $HOME/.bashrc

  install-clang-8:
    description: install clang-8 from backports
    steps:
      - run:
          name: add backports and install clang-8
          command: |
            echo 'deb http://deb.debian.org/debian buster-backports main' > /etc/apt/sources.list.d/backports.list
            apt-get update
            apt-get -qq -t buster-backports --no-install-recommends install clang-8 llvm-8

  # FIXME: the build-essential wart below is misformatted intentionally to remind us to replace the remotebackend testing deps with Debian packages
  auth-regress-setup:
    description: Prepare the environment for auth regression tests
    steps:
      - get-workspace
      - install-auth-deps
      - install-clang-8 # for the symbolizer
      - run:
          name: Install test dependencies
          command: |
            apt-get -y -qq install \
              authbind \
              bc \
              bind9utils \
              build-essential libsqlite3-dev libzmq3-dev \
              curl \
              default-jre-headless \
              dnsutils \
              gawk \
              git \
              ldnsutils \
              libnet-dns-perl \
              pdns-recursor \
              socat \
              sqlite3 \
              unbound-host
      - run:
          name: Install jdnssectools
          command: |
            if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then
              apt-get install -qq -y wget
              wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz
              tar xfz jdnssec-tools-0.14.tar.gz -C $HOME
              rm jdnssec-tools-0.14.tar.gz
            fi
            echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV
      - run:
          name: Allow missing tools in verify-dnssec-zone
          command: touch regression-tests/tests/verify-dnssec-zone/allow-missing
      - run:
          name: Start PowerDNS Recursor in the background
          command: pdns_recursor
          background: true
      - run:
          name: Set up authbind
          command: |
            touch /etc/authbind/byport/53
            chmod 755 /etc/authbind/byport/53

  auth-regress:
    description: Run one auth regression context
    parameters:
      skip:
        type: string
        default: ""
      rootskip:
        type: string
        default: ""
      doroot:
        type: boolean
        default: true
      context:
        type: string
        default: ""
      prefix:
        type: string
        default: "/opt/pdns-auth/"
    steps:
      - run:
          workdir: ~/project/regression-tests
          name: Run << parameters.context >> tests
          command: |
            [ -e ./vars ] && . ./vars
            export
            rm -rf tests/*/skip
            for t in << parameters.skip >>
            do
              touch tests/$t/skip
            done
            PDNS=<< parameters.prefix >>sbin/pdns_server \
            PDNS2=<< parameters.prefix >>sbin/pdns_server \
            SDIG=<< parameters.prefix >>bin/sdig \
            NOTIFY=<< parameters.prefix >>bin/pdns_notify \
            NSEC3DIG=<< parameters.prefix >>bin/nsec3dig \
            SAXFR=<< parameters.prefix >>bin/saxfr \
            ZONE2SQL=<< parameters.prefix >>bin/zone2sql \
            ZONE2LDAP=<< parameters.prefix >>bin/zone2ldap \
            PDNSUTIL=<< parameters.prefix >>bin/pdnsutil \
            PDNSCONTROL=<< parameters.prefix >>bin/pdns_control \
            RESOLVERIP=127.0.0.1 \
            ./start-test-stop 5300 << parameters.context >>
      - when:
          condition: << parameters.doroot >>
          steps:
            - run:
                workdir: ~/project/regression-tests.rootzone
                name: Run << parameters.context >> tests for the root zone
                command: |
                  [ -e ../regression-tests/vars ] && . ../regression-tests/vars
                  rm -rf tests/*/skip
                  for t in << parameters.rootskip >>
                  do
                    touch tests/$t/skip
                  done
                  PDNS=<< parameters.prefix >>sbin/pdns_server \
                  PDNS2=<< parameters.prefix >>sbin/pdns_server \
                  SDIG=<< parameters.prefix >>bin/sdig \
                  NOTIFY=<< parameters.prefix >>bin/pdns_notify \
                  NSEC3DIG=<< parameters.prefix >>bin/nsec3dig \
                  SAXFR=<< parameters.prefix >>bin/saxfr \
                  ZONE2SQL=<< parameters.prefix >>bin/zone2sql \
                  ZONE2LDAP=<< parameters.prefix >>bin/zone2ldap \
                  PDNSUTIL=<< parameters.prefix >>bin/pdnsutil \
                  PDNSCONTROL=<< parameters.prefix >>bin/pdns_control \
                  RESOLVERIP=127.0.0.1 \
                  ./start-test-stop 5300 << parameters.context >>

  install-auth-deps:
    description: Install all libraries needed to run the auth
    steps:
      - run: |
          apt-get install -qq -y --no-install-recommends \
            default-libmysqlclient-dev \
            libboost-all-dev \
            libcdb1 \
            libcurl4 \
            libgeoip1 \
            libkrb5-3 \
            libldap-2.4-2 \
            liblmdb0 \
            libluajit-5.1-2 \
            libmaxminddb0 \
            libp11-kit0 \
            libpq5 \
            libsodium23 \
            libssl1.1 \
            libsystemd0 \
            libyaml-cpp0.6 \
            softhsm2 \
            unixodbc

  install-dnsdist-deps:
    description: "Install all libraries needed for testing dnsdist"
    steps:
      - run: apt-get update
      - run:
          command: |
            apt-get install -qq -y \
              libluajit-5.1-2 \
              libboost-all-dev \
              libcap2 \
              libcdb1 \
              libcurl4-openssl-dev \
              libfstrm0 \
              libgnutls30 \
              libh2o-evloop0.13 \
              liblmdb0 \
              libnghttp2-dev \
              libre2-5 \
              libssl-dev \
              libsystemd0 \
              libsodium23 \
              patch \
              protobuf-compiler \
              python3-venv

  install-auth-dev-deps:
    description: Install all packages needed to build the auth
    steps:
      - run: apt-get update
      - run:
          command: |
            apt-get install -qq -y --no-install-recommends \
              autoconf \
              automake \
              bison \
              bzip2 \
              curl \
              default-libmysqlclient-dev \
              flex \
              g++ \
              git \
              libboost-all-dev \
              libcdb-dev \
              libcurl4-openssl-dev \
              libgeoip-dev \
              libkrb5-dev \
              libldap2-dev \
              liblmdb-dev \
              libluajit-5.1-dev \
              libmaxminddb-dev \
              libp11-kit-dev \
              libpq-dev \
              libsodium-dev \
              libsqlite3-dev \
              libssl-dev \
              libsystemd-dev \
              libtool \
              libyaml-cpp-dev \
              libzmq3-dev \
              make \
              pkg-config \
              python3-venv \
              ragel \
              ruby-bundler \
              ruby2.5-dev \
              sqlite3 \
              systemd \
              unixodbc-dev \
              wget

  add-auth-repo:
    description: "Add the debian repo for the auth"
    parameters:
      version:
        type: string
        default: "45"
      distro:
        type: string
        default: "debian"
      release:
        type: string
        default: "buster"
    steps:
      - run:
          name: "Install dependencies"
          command: "apt-get update && apt-get install -qq -y curl gnupg2"
      - run:
          name: "Install apt-keys"
          command: |
            if [ "<< parameters.version >>" = "master" ]; then
              curl https://repo.powerdns.com/CBC8B383-pub.asc -o /etc/apt/trusted.gpg.d/CBC8B383-pub.asc
            else
              curl https://repo.powerdns.com/FD380FBB-pub.asc -o /etc/apt/trusted.gpg.d/FD380FBB-pub.asc
            fi
      - run:
          name: "Add repository"
          command: |
            echo 'deb [arch=amd64] http://repo.powerdns.com/<< parameters.distro >> << parameters.release >>-auth-<< parameters.version>> main' >> /etc/apt/sources.list.d/pdns.list
      - run:
          name: "add pinning"
          command: |
            echo 'Package: pdns-*' > /etc/apt/preferences.d/pdns
            echo 'Pin: origin repo.powerdns.com' >> /etc/apt/preferences.d/pdns
            echo 'Pin-Priority: 600' >> /etc/apt/preferences.d/pdns
      - run:
          name: update apt cache
          command: apt-get update

  setup-ccache:
    description: Install .ccache and set $PATH
    steps:
      - run:
          name: Install ccache
          command: apt-get update && apt-get -qq -y install ccache
      - run:
          name: Setup ccache in $PATH
          command: echo 'export PATH=/usr/lib/ccache:$PATH' >> $BASH_ENV

  save-ccache-cache:
    description: Store the .ccache directory
    parameters:
      product:
        type: string
        default: ""
    steps:
      - save_cache:
          key: ccache-cache-{{ arch }}-<< parameters.product >>-{{ .Branch }}
          paths:
            - ~/.ccache

  restore-cache-ccache:
    description: Restore the .ccache directory
    parameters:
      product:
        type: string
        default: ""
    steps:
      - setup-ccache
      - run:
          name: ensure ca-certificates is installed
          command: apt-get update && apt-get -qq -y install ca-certificates
      - restore_cache:
          keys:
            - ccache-cache-{{ arch }}-<< parameters.product >>-{{ .Branch }}
            - ccache-cache-{{ arch }}-<< parameters.product >>-

jobs:
  checkout:
    resource_class: small

    docker:
      - image: debian:buster
        auth:
          username: powerdnsreadonly
          password: $DOCKERHUB_PASSWORD

    steps:
      - checkout-shallow
      - persist_to_workspace:
          root: /opt
          paths:
            - project

  build-auth:
    docker:
      - image: debian:buster
        auth:
          username: powerdnsreadonly
          password: $DOCKERHUB_PASSWORD
        environment:
          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
          ASAN_OPTIONS: detect_leaks=0
    steps:
      - get-workspace
      - install-auth-dev-deps
      - install-clang-8
      - restore-cache-ccache:
          product: auth
      - run:
          name: autoconf
          command: BUILDER_VERSION=0.0.0-git1 autoreconf -vfi
      - run:
          name: configure
          command: |
            CC='clang-8' \
            CXX='clang++-8' \
            CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
            CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
            ./configure \
              --enable-option-checking=fatal \
              --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' \
              --enable-systemd \
              --enable-tools \
              --enable-unit-tests \
              --enable-backend-unit-tests \
              --enable-fuzz-targets \
              --enable-experimental-pkcs11 \
              --enable-remotebackend-zeromq \
              --with-lmdb=/usr \
              --with-libsodium \
              --prefix=/opt/pdns-auth \
              --enable-ixfrdist \
              --enable-asan \
              --enable-ubsan
      - run:
          name: build
          command: make -j3 -k
      - save-ccache-cache:
          product: auth
      - run:
          name: Install the binaries
          command: make install
      - persist_to_workspace:
          root: /opt
          paths:
            - pdns-auth

  test-auth-regress-odbc-sqlite3:
    resource_class: small

    docker:
      - image: debian:buster
        auth:
          username: powerdnsreadonly
          password: $DOCKERHUB_PASSWORD
        environment:
          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
          ASAN_OPTIONS: detect_leaks=0
    steps:
      - auth-regress-setup
      - run:
          name: Configure ODBC for sqlite
          command: |
            cat >> ~/.odbc.ini \<<- __EOF__
            [pdns-sqlite3-1]
            Driver = SQLite3
            Database = ${PWD}/regression-tests/pdns.sqlite3
            [pdns-sqlite3-2]
            Driver = SQLite3
            Database = ${PWD}/regression-tests/pdns.sqlite32
            __EOF__
      - run:
          name: Install ODBC deps
          command: |
            apt-get install -qq -y \
            unixodbc \
            libsqliteodbc
      - run:
          name: Set up sqlite3 odbc testing
          command: echo 'export GODBC_SQLITE3_DSN=pdns-sqlite3-1' > ./vars
          workdir: ~/project/regression-tests
      - auth-regress:
          context: godbc_sqlite3-nsec3
          doroot: false # Broken at the moment

  test-auth-regress-odbc-mssql:
    docker:
      - image: debian:buster
        auth:
          username: powerdnsreadonly
          password: $DOCKERHUB_PASSWORD
        environment:
          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
          ASAN_OPTIONS: detect_leaks=0
      - image: mcr.microsoft.com/mssql/server:2017-GA-ubuntu
        environment:
          - ACCEPT_EULA: Y
          - SA_PASSWORD: 'SAsa12%%'
    steps:
      - auth-regress-setup
      - run:
          name: Install ODBC deps
          command: |
            apt-get install -qq -y \
            freetds-bin \
            tdsodbc \
            unixodbc
      - run:
          name: set up mssql odbc
          command: |
            cat >> ~/.odbc.ini \<<- __EOF__
            [pdns-mssql-docker]
            Driver=FreeTDS
            Trace=No
            Server=127.0.0.1
            Port=1433
            Database=pdns
            TDS_Version=7.1
            [pdns-mssql-docker-nodb]
            Driver=FreeTDS
            Trace=No
            Server=127.0.0.1
            Port=1433
            TDS_Version=7.1
            __EOF__
      - run:
          command: cat /usr/share/tdsodbc/odbcinst.ini <(echo Threading=1) >> /etc/odbcinst.ini
      - run:
          name: create database
          command: echo 'create database pdns' | isql -v pdns-mssql-docker-nodb sa SAsa12%%
      - run:
          name: Set up mssql odbc testing
          command: echo 'export GODBC_MSSQL_PASSWORD=SAsa12%% GODBC_MSSQL_USERNAME=sa GODBC_MSSQL_DSN=pdns-mssql-docker' > ./vars
          workdir: ~/project/regression-tests
      - auth-regress:
          context: godbc_mssql-nodnssec
          skip: 8bit-txt-unescaped
      - auth-regress:
          context: godbc_mssql
          skip: 8bit-txt-unescaped
      - auth-regress:
          context: godbc_mssql-nsec3
          skip: 8bit-txt-unescaped
      - auth-regress:
          context: godbc_mssql-nsec3-optout
          skip: 8bit-txt-unescaped
      - auth-regress:
          context: godbc_mssql-nsec3-narrow
          skip: 8bit-txt-unescaped

  test-auth-regress-bind:
    resource_class: small

    docker:
      - image: debian:buster
        auth:
          username: powerdnsreadonly
          password: $DOCKERHUB_PASSWORD
        environment:
          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
          ASAN_OPTIONS: detect_leaks=0
      - image: mysql:5      # for the hybrid test
        environment:
          MYSQL_ALLOW_EMPTY_PASSWORD: 1
        auth:
          username: powerdnsreadonly
          password: $DOCKERHUB_PASSWORD
    steps:
      - auth-regress-setup
      - run:
          command: apt-get install -qq -y p11-kit softhsm2
      - auth-regress:
          context: bind-dnssec-pkcs11
      - run:
          command: apt-get install -qq -y default-mysql-client
      - run:
          command: |
            cat >> ~/.my.cnf \<<- __EOF__
             [client]
             protocol=TCP
            __EOF__
      - run:
          name: Set up mysql client lib to force TCP
          command: echo 'export GMYSQLHOST=127.0.0.1' > ./vars
          workdir: ~/project/regression-tests
      - auth-regress:
          context: bind-hybrid-nsec3

  test-auth-regress-ldap:
    resource_class: small

    docker:
      - image: debian:buster
        auth:
          username: powerdnsreadonly
          password: $DOCKERHUB_PASSWORD
        environment:
          LDAPHOST: ldap://ldapserver/
          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
          ASAN_OPTIONS: detect_leaks=0
      - image: powerdns/ldap-regress:1.2.4-1 # OpenLDAP 2.4.47
        auth:
          username: powerdnsreadonly
          password: $DOCKERHUB_PASSWORD
        name: ldapserver
        command: '--loglevel debug'
        environment:
          LDAP_LOG_LEVEL: 0
    steps:
      - auth-regress-setup
      - run: DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ldap-utils
      - auth-regress:
          context: ldap-tree
          doroot: false
      - auth-regress:
          context: ldap-simple
          doroot: false
      - auth-regress:
          context: ldap-strict
          doroot: false

  test-auth-regress-geoip:
    resource_class: small

    docker:
      - image: debian:buster
        auth:
          username: powerdnsreadonly
          password: $DOCKERHUB_PASSWORD
        environment:
          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
          ASAN_OPTIONS: detect_leaks=0
    steps:
      - auth-regress-setup
      - run: export geoipdatabase=../modules/geoipbackend/regression-tests/GeoLiteCity.mmdb
      - auth-regress:
          context: geoip
          doroot: false

workflows:
  version: 2

  build-and-test-all:
    jobs:
      - checkout
      - build-auth:
          requires:
            - checkout
      - test-auth-regress-odbc-sqlite3:
          requires:
            - build-auth
      - test-auth-regress-odbc-mssql:
          requires:
            - build-auth
      - test-auth-regress-geoip:
          requires:
            - build-auth
      - test-auth-regress-ldap:
          requires:
            - build-auth