From: Remi Gacogne Date: Mon, 14 Oct 2024 14:14:57 +0000 (+0200) Subject: dnsdist: Add meson-based builds to the CI X-Git-Tag: dnsdist-2.0.0-alpha1~128^2~12 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=73689fb8bb8ba90ac839994c82d298d1b6c1dfe9;p=thirdparty%2Fpdns.git dnsdist: Add meson-based builds to the CI --- diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml index f7ce113e02..6593d9c899 100644 --- a/.github/workflows/build-and-test-all.yml +++ b/.github/workflows/build-and-test-all.yml @@ -216,11 +216,14 @@ jobs: needs: get-runner-container-image strategy: matrix: + builder: [autotools, meson] sanitizers: [ubsan+asan, tsan] features: [least, full] exclude: - sanitizers: tsan features: least + - sanitizers: tsan + builder: meson container: image: "${{ needs.get-runner-container-image.outputs.id }}:${{ needs.get-runner-container-image.outputs.tag }}" env: @@ -250,8 +253,8 @@ jobs: uses: actions/cache@v4 with: path: ~/.ccache - key: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-ccache-${{ steps.get-stamp.outputs.stamp }} - restore-keys: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-ccache- + key: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-${{ matrix.builder}}-ccache-${{ steps.get-stamp.outputs.stamp }} + restore-keys: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-${{ matrix.builder}}-ccache- - run: inv install-lld-linker-if-needed working-directory: ./pdns/dnsdistdist/ - run: inv ci-install-rust ${{ env.REPO_HOME }} @@ -259,18 +262,21 @@ jobs: - run: inv ci-build-and-install-quiche ${{ env.REPO_HOME }} working-directory: ./pdns/dnsdistdist/ - run: inv ci-autoconf + if: ${{ matrix.builder == 'autotools' }} working-directory: ./pdns/dnsdistdist/ - - run: inv ci-dnsdist-configure ${{ matrix.features }} + - run: inv ci-dnsdist-configure ${{ matrix.features }} ${{ matrix.builder }} dnsdist-${{ env.BUILDER_VERSION }} working-directory: ./pdns/dnsdistdist/ - run: inv ci-make-distdir + if: ${{ matrix.builder == 'autotools' }} working-directory: ./pdns/dnsdistdist/ - - run: inv ci-dnsdist-configure ${{ matrix.features }} - - run: inv ci-dnsdist-make-bear - - run: inv ci-dnsdist-run-unit-tests + - run: inv ci-dnsdist-configure ${{ matrix.features }} ${{ matrix.builder }} dnsdist-${{ env.BUILDER_VERSION }} + if: ${{ matrix.builder == 'autotools' }} + - run: inv ci-dnsdist-make-bear ${{ matrix.builder }} + - run: inv ci-dnsdist-run-unit-tests ${{ matrix.builder }} - run: inv generate-coverage-info ./testrunner $GITHUB_WORKSPACE - if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }} + if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' && matrix.builder == 'autotools'}} - name: Coveralls Parallel dnsdist unit - if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }} + if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' && matrix.builder == 'autotools' }} uses: coverallsapp/github-action@v2 with: flag-name: dnsdist-unit-${{ matrix.features }}-${{ matrix.sanitizers }} @@ -279,12 +285,14 @@ jobs: allow-empty: true fail-on-error: false - run: inv ci-make-install + if: ${{ matrix.builder == 'autotools' }} - run: ccache -s - run: echo "normalized-branch-name=${{ inputs.branch-name || github.ref_name }}" | tr "/" "-" >> "$GITHUB_ENV" - name: Store the binaries + if: ${{ matrix.builder == 'autotools' }} uses: actions/upload-artifact@v4 # this takes 30 seconds, maybe we want to tar with: - name: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-${{ env.normalized-branch-name }} + name: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-${{ matrix.builder}}-${{ env.normalized-branch-name }} path: /opt/dnsdist retention-days: 1 @@ -736,7 +744,7 @@ jobs: - name: Fetch the binaries uses: actions/download-artifact@v4 with: - name: dnsdist-full-${{ matrix.sanitizers }}-${{ env.normalized-branch-name }} + name: dnsdist-full-${{ matrix.sanitizers }}-autotools-${{ env.normalized-branch-name }} path: /opt/dnsdist - run: inv install-clang-runtime - run: inv install-dnsdist-test-deps $([ "$(. /etc/os-release && echo $VERSION_CODENAME)" = "bullseye" ] && echo "--skipXDP=True") diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 82d29cb0f7..54fb9a791d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -169,12 +169,12 @@ jobs: if: matrix.product == 'dnsdist' working-directory: ./pdns/dnsdistdist/ run: | - inv ci-dnsdist-configure full + inv ci-dnsdist-configure full autotools build-dir - name: Build dnsdist if: matrix.product == 'dnsdist' working-directory: ./pdns/dnsdistdist/ run: | - inv ci-dnsdist-make-bear + inv ci-dnsdist-make-bear autotools - run: ln -s ../../.clang-tidy.full .clang-tidy if: matrix.product == 'dnsdist' working-directory: ./pdns/dnsdistdist/ diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 48a1b1cecd..4433da6bad 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -84,7 +84,7 @@ jobs: working-directory: ./pdns/dnsdistdist/ - run: inv ci-build-and-install-quiche ${{ env.REPO_HOME }} working-directory: ./pdns/dnsdistdist/ - - run: inv ci-dnsdist-configure full + - run: inv ci-dnsdist-configure full autotools build-dir working-directory: ./pdns/dnsdistdist/ - run: inv coverity-make working-directory: ./pdns/dnsdistdist/ diff --git a/tasks.py b/tasks.py index f40080069c..c4de112ed0 100644 --- a/tasks.py +++ b/tasks.py @@ -351,6 +351,7 @@ def install_rec_build_deps(c, meson=False): @task(optional=['skipXDP']) def install_dnsdist_build_deps(c, skipXDP=False): c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps + (dnsdist_xdp_build_deps if not skipXDP else []))) + install_meson(c) @task def ci_autoconf(c, meson=False): @@ -662,11 +663,58 @@ def ci_rec_configure(c, features, build_dir=None, meson=False): ci_rec_configure_autotools(c, features) @task -def ci_dnsdist_configure(c, features): +def ci_dnsdist_configure(c, features, builder, build_dir): additional_flags = '' additional_ld_flags = '' if is_compiler_clang(): additional_ld_flags += '-fuse-ld=lld ' + + if features == 'least': + additional_flags = '-DDISABLE_COMPLETION \ + -DDISABLE_DELAY_PIPE \ + -DDISABLE_DYNBLOCKS \ + -DDISABLE_PROMETHEUS \ + -DDISABLE_PROTOBUF \ + -DDISABLE_BUILTIN_HTML \ + -DDISABLE_CARBON \ + -DDISABLE_SECPOLL \ + -DDISABLE_DEPRECATED_DYNBLOCK \ + -DDISABLE_LUA_WEB_HANDLERS \ + -DDISABLE_NON_FFI_DQ_BINDINGS \ + -DDISABLE_POLICIES_BINDINGS \ + -DDISABLE_PACKETCACHE_BINDINGS \ + -DDISABLE_DOWNSTREAM_BINDINGS \ + -DDISABLE_COMBO_ADDR_BINDINGS \ + -DDISABLE_CLIENT_STATE_BINDINGS \ + -DDISABLE_QPS_LIMITER_BINDINGS \ + -DDISABLE_SUFFIX_MATCH_BINDINGS \ + -DDISABLE_NETMASK_BINDINGS \ + -DDISABLE_DNSNAME_BINDINGS \ + -DDISABLE_DNSHEADER_BINDINGS \ + -DDISABLE_RECVMMSG \ + -DDISABLE_WEB_CACHE_MANAGEMENT \ + -DDISABLE_WEB_CONFIG \ + -DDISABLE_RULES_ALTERING_QUERIES \ + -DDISABLE_ECS_ACTIONS \ + -DDISABLE_TOP_N_BINDINGS \ + -DDISABLE_OCSP_STAPLING \ + -DDISABLE_HASHED_CREDENTIALS \ + -DDISABLE_FALSE_SHARING_PADDING \ + -DDISABLE_NPN' + + if builder == 'meson': + cmd = ci_dnsdist_configure_meson(features, additional_flags, build_dir) + logfile = 'meson-logs/meson-log.txt' + else: + cmd = ci_dnsdist_configure_autotools(features, additional_flags) + logfile = 'config.log' + + res = c.run(cmd, warn=True) + if res.exited != 0: + c.run(f'cat {logfile}') + raise UnexpectedExit(res) + +def ci_dnsdist_configure_autotools(features, additional_flags): if features == 'full': features_set = '--enable-dnstap \ --enable-dnscrypt \ @@ -700,41 +748,10 @@ def ci_dnsdist_configure(c, features): --without-net-snmp \ --without-nghttp2 \ --without-re2' - additional_flags = '-DDISABLE_COMPLETION \ - -DDISABLE_DELAY_PIPE \ - -DDISABLE_DYNBLOCKS \ - -DDISABLE_PROMETHEUS \ - -DDISABLE_PROTOBUF \ - -DDISABLE_BUILTIN_HTML \ - -DDISABLE_CARBON \ - -DDISABLE_SECPOLL \ - -DDISABLE_DEPRECATED_DYNBLOCK \ - -DDISABLE_LUA_WEB_HANDLERS \ - -DDISABLE_NON_FFI_DQ_BINDINGS \ - -DDISABLE_POLICIES_BINDINGS \ - -DDISABLE_PACKETCACHE_BINDINGS \ - -DDISABLE_DOWNSTREAM_BINDINGS \ - -DDISABLE_COMBO_ADDR_BINDINGS \ - -DDISABLE_CLIENT_STATE_BINDINGS \ - -DDISABLE_QPS_LIMITER_BINDINGS \ - -DDISABLE_SUFFIX_MATCH_BINDINGS \ - -DDISABLE_NETMASK_BINDINGS \ - -DDISABLE_DNSNAME_BINDINGS \ - -DDISABLE_DNSHEADER_BINDINGS \ - -DDISABLE_RECVMMSG \ - -DDISABLE_WEB_CACHE_MANAGEMENT \ - -DDISABLE_WEB_CONFIG \ - -DDISABLE_RULES_ALTERING_QUERIES \ - -DDISABLE_ECS_ACTIONS \ - -DDISABLE_TOP_N_BINDINGS \ - -DDISABLE_OCSP_STAPLING \ - -DDISABLE_HASHED_CREDENTIALS \ - -DDISABLE_FALSE_SHARING_PADDING \ - -DDISABLE_NPN' unittests = get_unit_tests() fuzztargets = get_fuzzing_targets() tools = f'''AR=llvm-ar-{clang_version} RANLIB=llvm-ranlib-{clang_version}''' if is_compiler_clang() else '' - configure_cmd = " ".join([ + return " ".join([ tools, get_base_configure_cmd(additional_c_flags='', additional_cxx_flags=additional_flags, additional_ld_flags=additional_ld_flags, enable_systemd=False, enable_sodium=False), features_set, @@ -744,10 +761,67 @@ def ci_dnsdist_configure(c, features): '--prefix=/opt/dnsdist' ]) - res = c.run(configure_cmd, warn=True) - if res.exited != 0: - c.run('cat config.log') - raise UnexpectedExit(res) +def ci_dnsdist_configure_meson(features, additional_flags, build_dir): + if features == 'full': + features_set = '-D cdb=enabled \ + -D dnscrypt=enabled \ + -D dnstap=enabled \ + -D ebpf=enabled \ + -D h2o=enabled \ + -D ipcipher=enabled \ + -D libedit=enabled \ + -D libsodium=enabled \ + -D lmdb=enabled \ + -D nghttp2=enabled \ + -D re2=enabled \ + -D systemd=enabled \ + -D tls-gnutls=enabled \ + -D dns-over-https=true \ + -D dns-over-http3=true \ + -D dns-over-quic=true \ + -D dns-over-tls=true \ + -D reproducible=true \ + -D snmp=true' + else: + features_set = '-D cdb=disabled \ + -D dnscrypt=disabled \ + -D dnstap=disabled \ + -D ebpf=disabled \ + -D h2o=disabled \ + -D ipcipher=disabled \ + -D libedit=disabled \ + -D libsodium=disabled \ + -D lmdb=disabled \ + -D nghttp2=disabled \ + -D re2=disabled \ + -D systemd=disabled \ + -D tls-gnutls=disabled \ + -D dns-over-https=false \ + -D dns-over-http3=false \ + -D dns-over-quic=false \ + -D dns-over-tls=false \ + -D reproducible=false \ + -D snmp=false' + unittests = get_unit_tests(meson=True) + fuzztargets = get_fuzzing_targets(meson=True) + tools = f'''AR=llvm-ar-{clang_version} RANLIB=llvm-ranlib-{clang_version}''' if is_compiler_clang() else '' + cflags = " ".join([get_cflags()]) + cxxflags = " ".join([get_cxxflags(), additional_flags]) + return " ".join([ + tools, + f'CFLAGS="{cflags}"', + f'CXXFLAGS="{cxxflags}"', + f"CC='{get_c_compiler()}'", + f"CXX='{get_cxx_compiler()}'", + f'. {repo_home}/.venv/bin/activate && meson setup {build_dir}', + features_set, + unittests, + fuzztargets, + "-D hardening-fortify-source=auto", + "-D auto-var-init=pattern", + get_coverage(meson=True), + get_sanitizers(meson=True) + ]) @task def ci_auth_make(c): @@ -783,8 +857,14 @@ def ci_rec_build(c, meson=False): def ci_dnsdist_make(c): c.run(f'make -j{get_build_concurrency(4)} -k V=1') +def ci_dnsdist_run_ninja(c): + c.run(f'. {repo_home}/.venv/bin/activate && ninja -j{get_build_concurrency()} --verbose') + @task -def ci_dnsdist_make_bear(c): +def ci_dnsdist_make_bear(c, builder): + if builder == 'meson': + return ci_dnsdist_run_ninja(c) + # Assumed to be running under ./pdns/dnsdistdist/ c.run(f'bear --append -- make -j{get_build_concurrency(4)} -k V=1') @@ -820,10 +900,16 @@ def ci_rec_run_unit_tests(c, meson=False): raise UnexpectedExit(res) @task -def ci_dnsdist_run_unit_tests(c): - res = c.run('make check', warn=True) +def ci_dnsdist_run_unit_tests(c, builder): + if builder == 'meson': + suite_timeout_sec = 120 + logfile = 'meson-logs/testlog.txt' + res = c.run(f'. {repo_home}/.venv/bin/activate && meson test --verbose -t {suite_timeout_sec}', warn=True) + else: + logfile = 'test-suite.log' + res = c.run('make check', warn=True) if res.exited != 0: - c.run('cat test-suite.log') + c.run(f'cat {logfile}', warn=True) raise UnexpectedExit(res) @task