From: Fred Morcos Date: Tue, 28 Feb 2023 14:49:40 +0000 (+0100) Subject: Github action to run clang-tidy on auth PRs X-Git-Tag: auth-4.8.0~15^2~10 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=fae3e64ca693ba54f36972dfd8e7c20f5fcf9117;p=thirdparty%2Fpdns.git Github action to run clang-tidy on auth PRs --- diff --git a/.github/scripts/clang-tidy.py b/.github/scripts/clang-tidy.py new file mode 100755 index 0000000000..85982e2079 --- /dev/null +++ b/.github/scripts/clang-tidy.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +"""Clang-tidy to Github Actions annotations converter. + +Convert the YAML file produced by clang-tidy-diff containing warnings and +suggested fixes to Github Actions annotations. + +""" + +import argparse +import os +import sys +from pathlib import Path + +import helpers + + +def create_argument_parser(): + """Create command-line argument parser.""" + parser = argparse.ArgumentParser( + description="Convert clang-tidy output to Github Actions" + ) + parser.add_argument( + "--fixes-file", + type=str, + required=True, + help="Path to the clang-tidy fixes YAML", + ) + return parser.parse_args() + + +def main(): + """Start the script.""" + args = create_argument_parser() + + fixes_path = Path(args.fixes_file) + compdb_filename = os.path.join(fixes_path.parent, "compile_commands.json") + compdb = helpers.load_compdb(compdb_filename) + compdb = helpers.index_compdb(compdb) + + fixes = helpers.load_fixes_file(args.fixes_file) + fixes = fixes["Diagnostics"] + have_warnings = False + for fix in fixes: + name = fix["DiagnosticName"] + level = fix["Level"] + directory = fix["BuildDirectory"] + diagnostic = fix["DiagnosticMessage"] + offset = diagnostic["FileOffset"] + filename = diagnostic["FilePath"] + message = diagnostic["Message"] + + if filename == "": + print(f"Meta error message from `{directory}`: {message}") + continue + + full_filename = filename + full_filename = Path(full_filename) + full_filename = ( + full_filename.as_posix() + if full_filename.is_absolute() + else os.path.join(directory, filename) + ) + + if full_filename not in compdb: + print( + f"Skipping `{full_filename}`" + " because it is not found" + " in the compilation database" + ) + continue + + try: + file_contents = helpers.load_file(full_filename) + except OSError: + # Skip in case the file can't be found. This is usually one of + # those "too many errors emitted, stopping now" clang messages. + print(f"Skipping `{full_filename}` because it is not found") + continue + + line = helpers.get_line_from_offset(file_contents, offset) + + annotation = "".join( + [ + f"::warning file={full_filename},line={line}", + f"::{message} ({name} - Level={level})", + ] + ) + print(annotation) + + # User-friendly printout + print(f"{level}: {full_filename}:{line}: {message} ({name})") + + have_warnings = True + + return 1 if have_warnings else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/helpers.py b/.github/scripts/helpers.py new file mode 100644 index 0000000000..cc170d1eed --- /dev/null +++ b/.github/scripts/helpers.py @@ -0,0 +1,48 @@ +"""Helpers for dealing with git, compilation databases, etc.""" + +import json +import os + +import git +import yaml + + +def load_file(filename): + """Load the entire contents of a file.""" + with open(filename, encoding="utf-8") as file: + contents = file.read() + return contents + + +def get_line_from_offset(file_contents, offset): + """Calculate line number from byte offset in source file.""" + return file_contents[:offset].count("\n") + 1 + + +def get_repo_root(): + """Get the git repo's root directory.""" + cwd = os.getcwd() + repo = git.Repo(cwd, search_parent_directories=True) + root = repo.git.rev_parse("--show-toplevel") + return root + + +def load_fixes_file(filename): + """Load the clang-tidy YAML fixes file.""" + with open(filename, encoding="utf_8") as file: + return yaml.safe_load(file) + + +def load_compdb(filename): + """Load the compilation database.""" + with open(filename, encoding="utf_8") as file: + return json.load(file) + + +def index_compdb(file_contents): + """Index the compilation database.""" + result = set() + for item in file_contents: + filename = os.path.join(item["directory"], item["file"]) + result.add(filename) + return result diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml new file mode 100644 index 0000000000..f4c8c7a130 --- /dev/null +++ b/.github/workflows/clang-tidy.yml @@ -0,0 +1,58 @@ +--- +name: 'clang-tidy' + +on: + pull_request: + branches: [master] + +permissions: + contents: read + +jobs: + clang-tidy: + name: auth clang-tidy + runs-on: ubuntu-20.04 + env: + UNIT_TESTS: yes + SANITIZERS: + steps: + - uses: PowerDNS/pdns/set-ubuntu-mirror@meta + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - name: get timestamp for cache + id: get-stamp + shell: bash + run: | + echo "stamp=$(/bin/date +%s)" >> "$GITHUB_OUTPUT" + - name: let GitHub cache our ccache data + uses: actions/cache@v3 + with: + path: ~/.ccache + key: auth-ccache-${{ steps.get-stamp.outputs.stamp }} + restore-keys: auth-ccache- + - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv install-clang + - run: inv install-clang-tidy-tools + - run: inv install-auth-build-deps + - run: inv ci-autoconf + - run: inv ci-auth-configure + - run: inv ci-auth-make-bear + - run: ccache -s + - run: mkdir clang-tidy-results + - run: ln -s .clang-tidy.full .clang-tidy + - name: Run clang-tidy + working-directory: pdns + run: git diff -U0 HEAD^ | python /usr/bin/clang-tidy-diff-12.py -p2 -export-fixes ../clang-tidy-results/auth-fixes.yml + - name: Print clang-tidy fixes YAML + shell: bash + run: | + if [ -f clang-tidy-results/auth-fixes.yml ]; then + cat clang-tidy-results/auth-fixes.yml + fi + - name: Result annotations + shell: bash + run: | + if [ -f clang-tidy-results/auth-fixes.yml ]; then + python .github/scripts/clang-tidy.py --fixes-file clang-tidy-results/auth-fixes.yml + fi diff --git a/build-scripts/gh-actions-setup-inv b/build-scripts/gh-actions-setup-inv index 53239df53b..0cd64feb5b 100755 --- a/build-scripts/gh-actions-setup-inv +++ b/build-scripts/gh-actions-setup-inv @@ -12,3 +12,5 @@ sudo apt-get autoremove sudo apt-get -qq -y --allow-downgrades dist-upgrade sudo apt-get -qq -y --no-install-recommends install python3-pip sudo pip3 install git+https://github.com/pyinvoke/invoke@faa5728a6f76199a3da1750ed952e7efee17c1da +sudo pip3 install gitpython +sudo pip3 install unidiff diff --git a/build-scripts/gh-actions-setup-inv-no-dist-upgrade b/build-scripts/gh-actions-setup-inv-no-dist-upgrade index be3eb520eb..e2e18aa60f 100755 --- a/build-scripts/gh-actions-setup-inv-no-dist-upgrade +++ b/build-scripts/gh-actions-setup-inv-no-dist-upgrade @@ -8,3 +8,5 @@ sudo chmod 755 /usr/sbin/policy-rc.d sudo apt-get update sudo apt-get -qq -y --no-install-recommends install python3-pip sudo pip3 install git+https://github.com/pyinvoke/invoke@faa5728a6f76199a3da1750ed952e7efee17c1da +sudo pip3 install gitpython +sudo pip3 install unidiff diff --git a/tasks.py b/tasks.py index 4608c6dd3c..23dc8be721 100644 --- a/tasks.py +++ b/tasks.py @@ -157,6 +157,10 @@ def install_clang(c): """ c.sudo('apt-get -qq -y --no-install-recommends install clang-12 llvm-12') +@task +def install_clang_tidy_tools(c): + c.sudo('apt-get -qq -y --no-install-recommends install clang-tidy-12 clang-tools-12 bear python-yaml') + @task def install_clang_runtime(c): # this gives us the symbolizer, for symbols in asan/ubsan traces @@ -525,6 +529,12 @@ def ci_dnsdist_configure(c, features): def ci_auth_make(c): c.run('make -j8 -k V=1') +@task +def ci_auth_make_bear(c): + # Needed for clang-tidy -line-filter vs project structure shenanigans + with c.cd('pdns'): + c.run('bear --append make -j8 -k V=1 -C ..') + @task def ci_rec_make(c): c.run('make -j8 -k V=1')