From: Remi Gacogne Date: Thu, 14 Sep 2023 13:06:13 +0000 (+0200) Subject: build-and-test-all: Generate code coverage data from CI X-Git-Tag: rec-5.0.0-alpha2~20^2~5 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=628a1dec1befdb7bc34053b5baf72e2d220e39f9;p=thirdparty%2Fpdns.git build-and-test-all: Generate code coverage data from CI --- diff --git a/.github/scripts/normalize_paths_in_coverage.py b/.github/scripts/normalize_paths_in_coverage.py new file mode 100755 index 0000000000..be0a80b122 --- /dev/null +++ b/.github/scripts/normalize_paths_in_coverage.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import os +import sys + +if __name__ == '__main__': + repositoryRoot = os.path.realpath(sys.argv[1]) + version = sys.argv[2] + inputFile = sys.argv[3] + outputFile = sys.argv[4] + with open(inputFile, mode='r') as inputFilePtr: + with open(outputFile, mode='w') as outputFilePtr: + for line in inputFilePtr: + if not line.startswith('SF:'): + outputFilePtr.write(line) + continue + + parts = line.split(':') + if len(parts) != 2: + outputFilePtr.write(line) + continue + + source_file = parts[1].rstrip() + # get rid of symbolic links + target = os.path.realpath(source_file) + + # get rid of the distdir path, to get file paths as they are in the repository + if f'pdns-{version}' in target: + # authoritative or tool + authPath = os.path.join(repositoryRoot, f'pdns-{version}') + relativeToAuth = os.path.relpath(target, authPath) + target = relativeToAuth + elif f'pdns-recursor-{version}' in target: + recPath = os.path.join(repositoryRoot, 'pdns', 'recursordist', f'pdns-recursor-{version}') + relativeToRec = os.path.relpath(target, recPath) + target = os.path.join('pdns', 'recursordist', relativeToRec) + elif f'dnsdist-{version}' in target: + distPath = os.path.join(repositoryRoot, 'pdns', 'dnsdistdist', f'dnsdist-{version}') + relativeToDist = os.path.relpath(target, distPath) + target = os.path.join('pdns', 'dnsdistdist', relativeToDist) + else: + print(f'Ignoring {target} that we could not map to a distdir', file=sys.stderr) + continue + + # we need to propery map symbolic links + fullPath = os.path.join(repositoryRoot, target) + if os.path.islink(fullPath): + # get the link target + realPath = os.path.realpath(fullPath) + # and make it relative again + target = os.path.relpath(realPath, repositoryRoot) + + outputFilePtr.write(f"SF:{target}\n") diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml index bd7d62b248..6e25bc0d12 100644 --- a/.github/workflows/build-and-test-all.yml +++ b/.github/workflows/build-and-test-all.yml @@ -15,6 +15,8 @@ env: # github.workspace variable points to the Runner home folder. Container home folder defined below. REPO_HOME: '/__w/pdns/pdns' BUILDER_VERSION: '0.0.0-git1' + COVERAGE: yes + LLVM_PROFILE_FILE: "/tmp/code-%p.profraw" jobs: build-auth: @@ -85,6 +87,15 @@ jobs: fi - run: inv ci-auth-install-remotebackend-test-deps - run: inv ci-auth-run-unit-tests + - run: inv generate-coverage-info ./testrunner $GITHUB_WORKSPACE + working-directory: ./pdns-${{ env.BUILDER_VERSION }}/pdns + - name: Coveralls Parallel auth unit + uses: coverallsapp/github-action@v2 + with: + flag-name: auth-unit-${{ matrix.sanitizers }} + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true - run: inv ci-make-install - run: ccache -s - name: Store the binaries @@ -162,6 +173,16 @@ jobs: echo "failed=$?" >> $GITHUB_OUTPUT fi - run: inv ci-rec-run-unit-tests + - run: inv generate-coverage-info ./testrunner $GITHUB_WORKSPACE + if: ${{ matrix.sanitizers != 'tsan' }} + - name: Coveralls Parallel rec unit + if: ${{ matrix.sanitizers != 'tsan' }} + uses: coverallsapp/github-action@v2 + with: + flag-name: rec-unit-${{ matrix.sanitizers }} + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true - run: inv ci-make-install - run: ccache -s - name: Store the binaries @@ -242,6 +263,16 @@ jobs: echo "failed=$?" >> $GITHUB_OUTPUT fi - run: inv ci-dnsdist-run-unit-tests + - run: inv generate-coverage-info ./testrunner $GITHUB_WORKSPACE + if: ${{ matrix.sanitizers != 'tsan' }} + - name: Coveralls Parallel dnsdist unit + if: ${{ matrix.sanitizers != 'tsan' }} + uses: coverallsapp/github-action@v2 + with: + flag-name: dnsdist-unit-${{ matrix.features }}-${{ matrix.sanitizers }} + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true - run: inv ci-make-install - run: ccache -s - name: Store the binaries @@ -301,6 +332,14 @@ jobs: - run: inv install-clang-runtime - run: inv install-auth-test-deps -b ${{ matrix.backend }} - run: inv test-api auth -b ${{ matrix.backend }} + - run: inv generate-coverage-info /opt/pdns-auth/sbin/pdns_server $GITHUB_WORKSPACE + - name: Coveralls Parallel auth API ${{ matrix.backend }} + uses: coverallsapp/github-action@v2 + with: + flag-name: auth-api-${{ matrix.backend }} + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true test-auth-backend: needs: build-auth @@ -413,6 +452,14 @@ jobs: - run: inv install-clang-runtime - run: inv install-auth-test-deps -b ${{ matrix.backend }} - run: inv test-auth-backend -b ${{ matrix.backend }} + - run: inv generate-coverage-info /opt/pdns-auth/sbin/pdns_server $GITHUB_WORKSPACE + - name: Coveralls Parallel auth backend ${{ matrix.backend }} + uses: coverallsapp/github-action@v2 + with: + flag-name: auth-backend-${{ matrix.backend }} + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true test-ixfrdist: needs: build-auth @@ -436,6 +483,14 @@ jobs: - run: inv install-clang-runtime - run: inv install-auth-test-deps - run: inv test-ixfrdist + - run: inv generate-coverage-info /opt/pdns-auth/bin/ixfrdist $GITHUB_WORKSPACE + - name: Coveralls Parallel ixfrdist + uses: coverallsapp/github-action@v2 + with: + flag-name: ixfrdist + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true test-recursor-api: needs: build-recursor @@ -468,6 +523,16 @@ jobs: - run: inv install-clang-runtime - run: inv install-rec-test-deps - run: inv test-api recursor + - run: inv generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE + if: ${{ matrix.sanitizers != 'tsan' }} + - name: Coveralls Parallel recursor API + if: ${{ matrix.sanitizers != 'tsan' }} + uses: coverallsapp/github-action@v2 + with: + flag-name: rec-api + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true test-recursor-regression: needs: build-recursor @@ -501,6 +566,16 @@ jobs: - run: inv install-clang-runtime - run: inv install-rec-test-deps - run: inv test-regression-recursor + - run: inv generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE + if: ${{ matrix.sanitizers != 'tsan' }} + - name: Coveralls Parallel recursor regression + if: ${{ matrix.sanitizers != 'tsan' }} + uses: coverallsapp/github-action@v2 + with: + flag-name: rec-regression + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true test-recursor-bulk: name: 'test rec *mini* bulk' @@ -532,6 +607,16 @@ jobs: - run: inv install-clang-runtime - run: inv install-rec-bulk-deps - run: inv test-bulk-recursor ${{ matrix.threads }} ${{ matrix.mthreads }} ${{ matrix.shards }} + - run: inv generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE + if: ${{ matrix.sanitizers != 'tsan' }} + - name: Coveralls Parallel recursor bulk + if: ${{ matrix.sanitizers != 'tsan' }} + uses: coverallsapp/github-action@v2 + with: + flag-name: rec-regression-bulk + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true test-dnsdist-regression: needs: build-dnsdist @@ -548,6 +633,8 @@ jobs: TSAN_OPTIONS: "halt_on_error=1:intercept_send=0:suppressions=${{ env.REPO_HOME }}/pdns/dnsdistdist/dnsdist-tsan.supp" # IncludeDir tests are disabled because of a weird interaction between TSAN and these tests which ever only happens on GH actions SKIP_INCLUDEDIR_TESTS: yes + SANITIZERS: ${{ matrix.sanitizers }} + COVERAGE: yes options: --sysctl net.ipv6.conf.all.disable_ipv6=0 steps: - uses: actions/checkout@v3 @@ -562,6 +649,16 @@ jobs: - run: inv install-clang-runtime - run: inv install-dnsdist-test-deps - run: inv test-dnsdist + - run: inv generate-coverage-info /opt/dnsdist/bin/dnsdist $GITHUB_WORKSPACE + if: ${{ matrix.sanitizers != 'tsan' }} + - name: Coveralls Parallel dnsdist regression + if: ${{ matrix.sanitizers != 'tsan' }} + uses: coverallsapp/github-action@v2 + with: + flag-name: dnsdist-regression-full-${{ matrix.sanitizers }} + path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov + parallel: true + allow-empty: true swagger-syntax-check: if: ${{ !github.event.schedule || vars.SCHEDULED_JOBS_BUILD_AND_TEST_ALL }} @@ -616,6 +713,10 @@ jobs: if: success() || failure() runs-on: ubuntu-20.04 steps: + - name: Coveralls Parallel Finished + uses: coverallsapp/github-action@v2 + with: + parallel-finished: true - name: Install jq and yq run: "sudo snap install jq yq" - name: Fail job if any of the previous jobs failed diff --git a/tasks.py b/tasks.py index 7d8982198c..97b7335247 100644 --- a/tasks.py +++ b/tasks.py @@ -203,6 +203,28 @@ def install_auth_build_deps(c): c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps)) install_libdecaf(c, 'pdns-auth') +def is_coverage_enabled(): + sanitizers = os.getenv('SANITIZERS') + if sanitizers: + sanitizers = sanitizers.split('+') + if 'tsan' in sanitizers: + return False + return os.getenv('COVERAGE') == 'yes' + +@task +def install_coverage_deps(c): + if is_coverage_enabled(): + c.sudo(f'apt-get install -y --no-install-recommends llvm-{clang_version}') + +@task +def generate_coverage_info(c, binary, outputDir): + if is_coverage_enabled(): + version = os.getenv('BUILDER_VERSION') + c.run(f'llvm-profdata-{clang_version} merge -sparse -o {outputDir}/temp.profdata /tmp/code-*.profraw') + c.run(f'llvm-cov-{clang_version} export --format=lcov --ignore-filename-regex=\'^/usr/\' -instr-profile={outputDir}/temp.profdata -object {binary} > {outputDir}/coverage.lcov') + c.run(f'{outputDir}/.github/scripts/normalize_paths_in_coverage.py {outputDir} {version} {outputDir}/coverage.lcov {outputDir}/normalized_coverage.lcov') + c.run(f'mv {outputDir}/normalized_coverage.lcov {outputDir}/coverage.lcov') + def setup_authbind(c): c.sudo('touch /etc/authbind/byport/53') c.sudo('chmod 755 /etc/authbind/byport/53') @@ -395,6 +417,7 @@ def ci_auth_configure(c): fuzz_targets = os.getenv('FUZZING_TARGETS') fuzz_targets = '--enable-fuzz-targets' if fuzz_targets == 'yes' else '' + coverage = '--enable-coverage=clang' if is_coverage_enabled() else '' modules = " ".join([ "bind", @@ -425,6 +448,7 @@ def ci_auth_configure(c): sanitizers, unittests, fuzz_targets, + coverage, ]) res = c.run(configure_cmd, warn=True) if res.exited != 0: @@ -438,6 +462,7 @@ def ci_rec_configure(c): unittests = os.getenv('UNIT_TESTS') unittests = '--enable-unit-tests' if unittests == 'yes' else '' + coverage = '--enable-coverage=clang' if is_coverage_enabled() else '' configure_cmd = " ".join([ get_base_configure_cmd(), @@ -449,6 +474,7 @@ def ci_rec_configure(c): "--enable-dns-over-tls", sanitizers, unittests, + coverage, ]) res = c.run(configure_cmd, warn=True) if res.exited != 0: @@ -523,8 +549,9 @@ def ci_dnsdist_configure(c, features): unittests = ' --enable-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else '' fuzztargets = '--enable-fuzz-targets' if os.getenv('FUZZING_TARGETS') == 'yes' else '' sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else '' - cflags = '-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int' - cxxflags = cflags + ' -Wp,-D_GLIBCXX_ASSERTIONS ' + additional_flags + coverage = '--enable-coverage=clang' if is_coverage_enabled() else '' + cflags = get_cflags() + cxxflags = " ".join([get_cxxflags(), additional_flags]) res = c.run(f'''CFLAGS="%s" \ CXXFLAGS="%s" \ AR=llvm-ar-{clang_version} \ @@ -536,7 +563,7 @@ def ci_dnsdist_configure(c, features): --enable-fortify-source=auto \ --enable-auto-var-init=pattern \ --enable-lto=thin \ - --prefix=/opt/dnsdist %s %s %s %s''' % (cflags, cxxflags, features_set, sanitizers, unittests, fuzztargets), warn=True) + --prefix=/opt/dnsdist %s %s %s %s %s''' % (cflags, cxxflags, features_set, sanitizers, unittests, fuzztargets, coverage), warn=True) if res.exited != 0: c.run('cat config.log') raise UnexpectedExit(res)