Merge pull request #7786 from pieterlexis/circle-ci-part1

[thirdparty/pdns.git] / .circleci / config.yml

version: 2.1

commands:
  checkout-shallow:
    description: "Do a shallow checkout of the repository"
    steps:
      - run:
          name: Install git and openssh
          command: apt-get update && apt-get -qq -y install openssh-client git
      - run:
          name: Store the keyid for the git repo
          command: mkdir -p $HOME/.ssh && ssh-keyscan $(echo ${CIRCLE_REPOSITORY_URL} | sed 's,.*@\([^:]*\):.*,\1,') > $HOME/.ssh/known_hosts
      - run:
          name: Clone the repo
          command: git clone --depth 1 --branch $CIRCLE_BRANCH $CIRCLE_REPOSITORY_URL ~/project

  auth-regress-setup:
    description: Prepare the environment for auth regression tests
    steps:
      - run:
          name: ensure ca-certificates is installed
          command: apt-get update && apt-get -qq -y install ca-certificates
      - attach_workspace:
          at: /opt
      - install-auth-deps
      - run:
          name: Install test dependencies
          command: |
            apt-get -y -qq install \
              bc \
              bind9utils \
              default-jre-headless \
              dnsutils \
              ldnsutils \
              libnet-dns-perl \
              unbound-host
      - run:
          name: Install jdnssectools
          command: |
            if [ ! -e /usr/bin/jdnssec-verifyzone ]; then
              apt-get install -qq -y wget
              wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz
              tar xfz jdnssec-tools-0.14.tar.gz --strip-components=1 -C /
              rm jdnssec-tools-0.14.tar.gz
            fi
      - checkout-shallow
      - run:
          name: Allow missing tools in verify-dnssec-zone
          command: touch regression-tests/tests/verify-dnssec-zone/allow-missing

  auth-regress:
    description: Run one auth regression context
    parameters:
      skip:
        type: string
        default: ""
      rootskip:
        type: string
        default: ""
      doroot:
        type: boolean
        default: true
      context:
        type: string
        default: ""
      prefix:
        type: string
        default: "/opt/pdns-auth/"
    steps:
      - run:
          workdir: ~/project/regression-tests
          name: Run << parameters.context >> tests
          command: |
            [ -e ./vars ] && . ./vars
            rm -rf tests/*/skip
            for t in << parameters.skip >>
            do
              touch tests/$t/skip
            done
            PDNS=<< parameters.prefix >>sbin/pdns_server \
            PDNS2=<< parameters.prefix >>sbin/pdns_server \
            SDIG=<< parameters.prefix >>bin/sdig \
            NOTIFY=<< parameters.prefix >>bin/pdns_notify \
            NSEC3DIG=<< parameters.prefix >>bin/nsec3dig \
            SAXFR=<< parameters.prefix >>bin/saxfr \
            ZONE2SQL=<< parameters.prefix >>bin/zone2sql \
            ZONE2LDAP=<< parameters.prefix >>bin/zone2ldap \
            PDNSUTIL=<< parameters.prefix >>bin/pdnsutil \
            PDNSCONTROL=<< parameters.prefix >>bin/pdns_control \
            ./start-test-stop 5300 << parameters.context >>
      - when:
          condition: << parameters.doroot >>
          steps:
            - run:
                workdir: ~/project/regression-tests.rootzone
                name: Run << parameters.context >> tests for the root zone
                command: |
                  [ -e ../regression-tests/vars ] && . ../regression-tests/vars
                  rm -rf tests/*/skip
                  for t in << parameters.rootskip >>
                  do
                    touch tests/$t/skip
                  done
                  PDNS=<< parameters.prefix >>sbin/pdns_server \
                  PDNS2=<< parameters.prefix >>sbin/pdns_server \
                  SDIG=<< parameters.prefix >>bin/sdig \
                  NOTIFY=<< parameters.prefix >>bin/pdns_notify \
                  NSEC3DIG=<< parameters.prefix >>bin/nsec3dig \
                  SAXFR=<< parameters.prefix >>bin/saxfr \
                  ZONE2SQL=<< parameters.prefix >>bin/zone2sql \
                  ZONE2LDAP=<< parameters.prefix >>bin/zone2ldap \
                  PDNSUTIL=<< parameters.prefix >>bin/pdnsutil \
                  PDNSCONTROL=<< parameters.prefix >>bin/pdns_control \
                  ./start-test-stop 5300 << parameters.context >>

  install-recursor-deps:
    description: "Install all libraries needed for the recursor"
    steps:
      - run: apt-get update
      - run:
          command: |
            apt-get install -qq -y \
              libluajit-5.1 \
              libboost-all-dev \
              libcap2 \
              libssl1.1 \
              libsystemd0 \
              libsodium18 \
              libprotobuf10

  install-auth-deps:
    description: Install all libraries needed to run the auth
    steps:
      - run: |
          apt-get install -qq -y --no-install-recommends \
            libboost-all-dev \
            libcdb1 \
            libkrb5-3 \
            libldap-2.4-2 \
            liblmdb0 \
            libpq5 \
            libssl1.1 \
            libsodium18 \
            libsystemd0 \
            default-libmysqlclient-dev \
            unixodbc

  install-auth-dev-deps:
    description: Install all packages needed to build the auth
    steps:
      - run: apt-get update
      - run:
          command: |
            apt-get install -qq -y --no-install-recommends \
              autoconf \
              automake \
              bison \
              default-libmysqlclient-dev \
              flex \
              g++ \
              git \
              libboost-all-dev \
              libcdb-dev \
              libkrb5-dev \
              libldap2-dev \
              liblmdb-dev \
              libpq-dev \
              libsodium-dev \
              libsqlite3-dev \
              libssl-dev \
              libtool \
              make \
              pkg-config \
              ragel \
              sqlite3 \
              unixodbc-dev \
              virtualenv \
              wget

  add-auth-repo:
    description: "Add the debian repo for the auth"
    parameters:
      version:
        type: string
        default: "42"
      distro:
        type: string
        default: "debian"
      release:
        type: string
        default: "stretch"
    steps:
      - run:
          name: "Install dependencies"
          command: "apt-get update && apt-get install -qq -y curl gnupg2"
      - run:
          name: "Install apt-keys"
          command: |
            if [ "<< parameters.version >>" = "master" ]; then
              curl https://repo.powerdns.com/CBC8B383-pub.asc | apt-key add -
            else
              curl https://repo.powerdns.com/FD380FBB-pub.asc | apt-key add -
            fi
      - run:
          name: "Add repository"
          command: |
            echo 'deb [arch=amd64] http://repo.powerdns.com/<< parameters.distro >> << parameters.release >>-auth-<< parameters.version>> main' >> /etc/apt/sources.list.d/pdns.list
      - run:
          name: "add pinning"
          command: |
            echo 'Package: pdns-*' > /etc/apt/preferences.d/pdns
            echo 'Pin: origin repo.powerdns.com' >> /etc/apt/preferences.d/pdns
            echo 'Pin-Priority: 600' >> /etc/apt/preferences.d/pdns
      - run:
          name: update apt cache
          command: apt-get update

  setup-ccache:
    description: Install .ccache and set $PATH
    steps:
      - run:
          name: Install ccache
          command: apt-get update && apt-get -qq -y install ccache
      - run:
          name: Setup ccache in $PATH
          command: echo 'export PATH=/usr/lib/ccache:$PATH' >> $BASH_ENV

  save-ccache-cache:
    description: Store the .ccache directory
    parameters:
      product:
        type: string
        default: ""
    steps:
      - save_cache:
          key: ccache-cache-{{ arch }}-<< parameters.product >>-{{ .Branch }}
          paths:
            - ~/.ccache

  restore-cache-ccache:
    description: Restore the .ccache directory
    parameters:
      product:
        type: string
        default: ""
    steps:
      - setup-ccache
      - run:
          name: ensure ca-certificates is installed
          command: apt-get update && apt-get -qq -y install ca-certificates
      - restore_cache:
          keys:
            - ccache-cache-{{ arch }}-<< parameters.product >>-{{ .Branch }}
            - ccache-cache-{{ arch }}-<< parameters.product >>-

jobs:
  build-auth:
    docker:
      - image: debian:stretch

    steps:
      - checkout-shallow
      - install-auth-dev-deps
      - restore-cache-ccache:
          product: auth
      - run:
          name: autoconf
          command: BUILDER_VERSION=0.0.0-git1 autoreconf -vfi
      - run:
          name: configure
          command: |
            CFLAGS="-O1 -Werror=vla" \
            CXXFLAGS="-O1 -Werror=vla" \
            ./configure \
              --disable-lua-records \
              --with-modules='bind lmdb ldap gmysql gsqlite3 gpgsql godbc mydns random tinydns' \
              --enable-tools \
              --with-lmdb=/usr \
              --with-libsodium \
              --prefix=/opt/pdns-auth
      - run:
          name: build
          command: make -j3 -k
      - save-ccache-cache:
          product: auth
      - run:
          name: Install the binaries
          command: make install
      - persist_to_workspace:
          root: /opt
          paths:
            - pdns-auth

  test-auth-regress-odbc-sqlite3:
    docker:
      - image: debian:stretch
    steps:
      - auth-regress-setup
      - run:
          name: Configure ODBC for sqlite
          command: |
            cat >> ~/.odbc.ini \<<- __EOF__
            [pdns-sqlite3-1]
            Driver = SQLite3
            Database = ${PWD}/regression-tests/pdns.sqlite3
            [pdns-sqlite3-2]
            Driver = SQLite3
            Database = ${PWD}/regression-tests/pdns.sqlite32
            __EOF__
      - run:
          name: Install ODBC deps
          command: |
            apt-get install -qq -y \
            unixodbc \
            libsqliteodbc \
            sqlite3
      - run:
          name: Set up sqlite3 odbc testing
          command: echo 'export GODBC_SQLITE3_DSN=pdns-sqlite3-1' > ./vars
          workdir: ~/project/regression-tests
      - auth-regress:
          context: godbc_sqlite3-nsec3
          doroot: false # Broken at the moment

  test-auth-regress-odbc-mssql:
    docker:
      - image: debian:stretch
      - image: mcr.microsoft.com/mssql/server:2019-CTP2.2-ubuntu
        environment:
          - ACCEPT_EULA: Y
          - SA_PASSWORD: 'SAsa12%%'
    steps:
      - auth-regress-setup
      - run:
          name: Install ODBC deps
          command: |
            apt-get install -qq -y \
            freetds-bin \
            tdsodbc \
            unixodbc
      - run:
          name: set up mssql odbc
          command: |
            cat >> ~/.odbc.ini \<<- __EOF__
            [pdns-mssql-docker]
            Driver=FreeTDS
            Trace=No
            Server=127.0.0.1
            Port=1433
            Database=pdns
            TDS_Version=7.1
            [pdns-mssql-docker-nodb]
            Driver=FreeTDS
            Trace=No
            Server=127.0.0.1
            Port=1433
            TDS_Version=7.1
            __EOF__
      - run:
          command: cat /usr/share/tdsodbc/odbcinst.ini >> /etc/odbcinst.ini
      - run:
          name: create database
          command: echo 'create database pdns' | isql -v pdns-mssql-docker-nodb sa SAsa12%%
      - run:
          name: Set up mssql odbc testing
          command: echo 'export GODBC_MSSQL_PASSWORD=SAsa12%% GODBC_MSSQL_USERNAME=sa GODBC_MSSQL_DSN=pdns-mssql-docker' > ./vars
          workdir: ~/project/regression-tests
      - auth-regress:
          context: godbc_mssql-nodnssec
          skip: 8bit-txt-unescaped
      - auth-regress:
          context: godbc_mssql
          skip: 8bit-txt-unescaped
      - auth-regress:
          context: godbc_mssql-nsec3
          skip: 8bit-txt-unescaped
      - auth-regress:
          context: godbc_mssql-nsec3-optout
          skip: 8bit-txt-unescaped
      - auth-regress:
          context: godbc_mssql-nsec3-narrow
          skip: 8bit-txt-unescaped

  test-auth-regress-gsqlite3:
    docker:
      - image: debian:stretch
    steps:
      - auth-regress-setup
      - run:
          command: apt-get install -qq -y sqlite3
      - auth-regress:
          context: gsqlite3-nodnssec-both
      - auth-regress:
          context: gsqlite3-both
      - auth-regress:
          context: gsqlite3-nsec3-both
      - auth-regress:
          context: gsqlite3-nsec3-optout-both
      - auth-regress:
          context: gsqlite3-nsec3-narrow

  test-auth-regress-bind:
    docker:
      - image: debian:stretch
      - image: circleci/mysql:5      # for the hybrid test
    steps:
      - auth-regress-setup
      - run:
          command: apt-get install -qq -y sqlite3
      - auth-regress:
          context: bind-both
      - auth-regress:
          context: bind-dnssec-both
      - auth-regress:
          context: bind-dnssec-nsec3-both
      - auth-regress:
          context: bind-dnssec-nsec3-optout-both
      - auth-regress:
          context: bind-dnssec-nsec3-narrow
      - run:
          command: apt-get install -qq -y default-mysql-client
      - run:
          command: |
            cat >> ~/.my.cnf \<<- __EOF__
             [client]
             protocol=TCP
            __EOF__
      - auth-regress:
          context: bind-hybrid-nsec3

  test-auth-regress-mydns:
    docker:
      - image: debian:stretch
      - image: circleci/mysql:5
    steps:
      - auth-regress-setup
      - run:
          command: apt-get install -qq -y default-mysql-client
      - run:
          command: |
            cat >> ~/.my.cnf \<<- __EOF__
             [client]
             protocol=TCP
            __EOF__
      - auth-regress:
          context: mydns
          doroot: false

  test-auth-regress-gmysql:
    docker:
      - image: debian:stretch
      - image: circleci/mysql:5
    steps:
      - auth-regress-setup
      - run:
          command: apt-get install -qq -y default-mysql-client
      - run:
          command: |
            cat >> ~/.my.cnf \<<- __EOF__
             [client]
             protocol=TCP
            __EOF__
      - auth-regress:
          context: gmysql-nodnssec-both
      - auth-regress:
          context: gmysql-both
      - auth-regress:
          context: gmysql-nsec3-both
      - auth-regress:
          context: gmysql-nsec3-optout-both
      - auth-regress:
          context: gmysql-nsec3-narrow

  test-auth-regress-gpgsql:
    docker:
      - image: debian:stretch
      - image: circleci/postgres:9
        environment:
          POSTGRES_USER: root
    steps:
      - auth-regress-setup
      - run:
          command: apt-get install -qq -y postgresql-client
      - run:
          name: Use TCP for Postgresql connections
          command: |
            cat >> ./vars \<<- __EOF__
            export PGHOST=127.0.0.1
            export PGPORT=5432
            __EOF__
          workdir: ~/project/regression-tests
      - auth-regress:
          context: gpgsql-nodnssec-both
      - auth-regress:
          context: gpgsql-both
      - auth-regress:
          context: gpgsql-nsec3-both
      - auth-regress:
          context: gpgsql-nsec3-optout-both
      - auth-regress:
          context: gpgsql-nsec3-narrow

  test-auth-regress-ldap:
    docker:
      - image: debian:stretch
        environment:
          LDAPHOST: ldap://ldapserver/
      - image: powerdns/ldap-regress:1.2.4-0 # OpenLDAP 2.4.47
        name: ldapserver
        command: '--loglevel debug'
        environment:
          LDAP_LOG_LEVEL: 0
    steps:
      - auth-regress-setup
      - run: DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ldap-utils
      - auth-regress:
          context: ldap-tree
          doroot: false
      - auth-regress:
          context: ldap-simple
          doroot: false
      - auth-regress:
          context: ldap-strict
          doroot: false

  test-auth-regress-tinydns:
    docker:
      - image: debian:stretch
    steps:
      - auth-regress-setup
      - auth-regress:
          context: tinydns
          doroot: false

  test-auth-regress-lmdb:
    docker:
      - image: debian:stretch
    steps:
      - auth-regress-setup
      - run: apt-get install -y -qq jq curl
      - auth-regress:
          context: lmdb-nodnssec-both
      - auth-regress:
          context: lmdb-both
      - auth-regress:
          context: lmdb-nsec3-both
      - auth-regress:
          context: lmdb-nsec3-optout-both
      - auth-regress:
          context: lmdb-nsec3-narrow

  test-auth-algorithms:
    docker:
      - image: debian:stretch
    steps:
      - auth-regress-setup
      - run:
          name: Test all algorithms
          command: /opt/pdns-auth/bin/pdnsutil test-algorithms

  test-auth-api:
    docker:
      - image: debian:stretch
    steps:
      - auth-regress-setup
      - run: apt-get -y -qq install virtualenv sqlite3
      - run:
          name: Run API tests
          workdir: ~/project/regression-tests.api
          command: |
            PDNSSERVER="/opt/pdns-auth/sbin/pdns_server" \
            PDNSUTIL="/opt/pdns-auth/bin/pdnsutil" \
            SDIG="/opt/pdns-auth/bin/sdig" \
            ZONE2SQL="/opt/pdns-auth/bin/zone2sql" \
            ./runtests authoritative

  build-recursor:
    docker:
      - image: debian:stretch
    steps:
      - restore-cache-ccache:
          product: recursor
      - run:
          name: Install dependencies
          command: |
            apt-get update && apt-get -qq --no-install-recommends install \
            autoconf \
            automake \
            ca-certificates \
            curl \
            bison \
            flex \
            g++ \
            git \
            libboost-all-dev \
            libcap-dev \
            libluajit-5.1-dev \
            libprotobuf-dev \
            libsodium-dev \
            libssl-dev \
            libsystemd-dev \
            libtool \
            make \
            pkg-config \
            protobuf-compiler \
            ragel \
            virtualenv
      - checkout-shallow
      - run:
          name: autoconf
          command: BUILDER_VERSION=0.0.0-git1 autoreconf -vfi
          working_directory: ~/project/pdns/recursordist
      - run:
          name: configure
          command: |
            CFLAGS="-O1 -Werror=vla" \
            CXXFLAGS="-O1 -Werror=vla" \
            ./configure \
            --enable-unit-tests \
            --prefix=/opt/pdns-recursor \
            --with-libsodium \
            --with-lua=luajit \
            --with-libcap \
            --with-protobuf=yes \
            --without-net-snmp
          working_directory: ~/project/pdns/recursordist
      - run:
          name: build
          command: make -j3 -k
          working_directory: ~/project/pdns/recursordist
      - save-ccache-cache:
          product: recursor
      - run:
          name: Run unit tests
          command: make check || (cat test-suite.log; false)
          working_directory: ~/project/pdns/recursordist
      - run:
          name: Install resulting binaries
          command: make install
          working_directory: ~/project/pdns/recursordist
      - persist_to_workspace:
          root: /opt
          paths:
            - pdns-recursor

  test-recursor-regression:
    docker:
      - image: debian:stretch
    steps:
      - add-auth-repo
      - run: apt-get --no-install-recommends install -qq -y pdns-server pdns-backend-bind pdns-tools daemontools authbind jq libfaketime lua-posix moreutils bc virtualenv protobuf-compiler
      - install-recursor-deps
      - run:
          name: Set up authbind
          command: |
            touch /etc/authbind/byport/53
            chmod 755 /etc/authbind/byport/53
      - checkout-shallow
      - attach_workspace:
          at: /opt
      - run:
          name: Run regression tests
          workdir: ~/project
          command: |
            PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \
            RECCONTROL=/opt/pdns-recursor/bin/rec_control \
            ./build-scripts/test-recursor

  test-recursor-bulk:
    docker:
      - image: debian:stretch
    steps:
      - add-auth-repo
      - run: apt-get --no-install-recommends install -qq -y pdns-tools moreutils time
      - install-recursor-deps
      - checkout-shallow
      - attach_workspace:
          at: /opt
      - run:
          name: Get the majestic million list
          workdir: ~/project/regression-tests
          command: |
            apt-get install -qq -y unzip && \
            curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip && \
            unzip top-1m.csv.zip -d .
      - run:
          name: Run bulktests
          command: |
            DNSBULKTEST=/usr/bin/dnsbulktest \
            RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \
            THRESHOLD=95 \
            TRACE=no \
            ./timestamp ./recursor-test 5300 50000 || \
            (cat recursor.log; false)
          workdir: ~/project/regression-tests

  test-recursor-api:
    docker:
      - image: debian:stretch
    steps:
      - add-auth-repo
      - run: apt-get --no-install-recommends install -qq -y virtualenv pdns-tools
      - install-recursor-deps
      - checkout-shallow
      - attach_workspace:
          at: /opt
      - run:
          name: Run API tests
          workdir: ~/project/regression-tests.api
          command: |
            PDNSRECURSOR="/opt/pdns-recursor/sbin/pdns_recursor" \
            ./runtests recursor

workflows:
  version: 2
  build-and-test-all:
    jobs:
      - build-auth
      - test-auth-api:
          requires:
            - build-auth
      - test-auth-algorithms:
          requires:
            - build-auth
      - test-auth-regress-odbc-sqlite3:
          requires:
            - build-auth
      - test-auth-regress-odbc-mssql:
          requires:
            - build-auth
      - test-auth-regress-gmysql:
          requires:
            - build-auth
      - test-auth-regress-mydns:
          requires:
            - build-auth
      - test-auth-regress-bind:
          requires:
            - build-auth
      - test-auth-regress-gsqlite3:
          requires:
            - build-auth
      - test-auth-regress-gpgsql:
          requires:
            - build-auth
      - test-auth-regress-lmdb:
          requires:
            - build-auth
      - test-auth-regress-ldap:
          requires:
            - build-auth
      - test-auth-regress-tinydns:
          requires:
            - build-auth
      - build-recursor
      - test-recursor-regression:
          requires:
            - build-recursor
      - test-recursor-bulk:
          requires:
            - build-recursor
      - test-recursor-api:
          requires:
            - build-recursor