From: Remi Gacogne Date: Fri, 13 Oct 2023 09:42:55 +0000 (+0200) Subject: Move clang-tidy analysis to the CodeQL workflow X-Git-Tag: rec-5.0.0-beta1~23^2~14 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2ef73ca0bfe2316b4bc2dd26057dfc7e3529db12;p=thirdparty%2Fpdns.git Move clang-tidy analysis to the CodeQL workflow --- diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml index 528c882072..22b31a6635 100644 --- a/.github/workflows/build-and-test-all.yml +++ b/.github/workflows/build-and-test-all.yml @@ -11,12 +11,14 @@ permissions: # least privileges, see https://docs.github.com/en/actions/using-wo contents: read env: + COMPILER: clang CLANG_VERSION: '13' # github.workspace variable points to the Runner home folder. Container home folder defined below. REPO_HOME: '/__w/pdns/pdns' BUILDER_VERSION: '0.0.0-git1' COVERAGE: yes LLVM_PROFILE_FILE: "/tmp/code-%p.profraw" + OPTIMIZATIONS: yes jobs: build-auth: @@ -35,8 +37,6 @@ jobs: defaults: run: working-directory: ./pdns-${{ env.BUILDER_VERSION }} - outputs: - clang-tidy-failed: ${{ steps.clang-tidy-annotations.outputs.failed }} steps: - uses: actions/checkout@v3 with: @@ -64,34 +64,6 @@ jobs: working-directory: . - run: inv ci-auth-configure - run: inv ci-auth-make-bear # This runs under pdns-$BUILDER_VERSION/pdns/ - - name: Normalize paths in compilation DB - working-directory: . - run: python3 .github/scripts/normalize_paths_in_compilation_database.py --version $BUILDER_VERSION pdns-$BUILDER_VERSION/pdns/compile_commands.json - - name: Copy the compilation DB - working-directory: . - run: cp pdns-$BUILDER_VERSION/pdns/compile_commands.json . - - run: ln -s .clang-tidy.full .clang-tidy - working-directory: . - - name: Run clang-tidy - working-directory: . - run: git diff -U0 HEAD^..HEAD | python3 .github/scripts/git-filter.py | python3 .github/scripts/clang-tidy-diff.py -clang-tidy-binary /usr/bin/clang-tidy-${CLANG_VERSION} -extra-arg=-ferror-limit=0 -p1 -export-fixes clang-tidy-auth.yml - - name: Print clang-tidy fixes YAML - working-directory: . - shell: bash - run: | - if [ -f clang-tidy-auth.yml ]; then - cat clang-tidy-auth.yml - fi - - name: Result annotations - id: clang-tidy-annotations - working-directory: . - shell: bash - run: | - if [ -f clang-tidy-auth.yml ]; then - set +e - python3 .github/scripts/clang-tidy.py --fixes-file clang-tidy-auth.yml - echo "failed=$?" >> $GITHUB_OUTPUT - fi - run: inv ci-auth-install-remotebackend-test-deps - run: inv ci-auth-run-unit-tests - run: inv generate-coverage-info ./testrunner $GITHUB_WORKSPACE @@ -130,8 +102,6 @@ jobs: defaults: run: working-directory: ./pdns/recursordist/pdns-recursor-${{ env.BUILDER_VERSION }} - outputs: - clang-tidy-failed: ${{ steps.clang-tidy-annotations.outputs.failed }} steps: - uses: actions/checkout@v3 with: @@ -161,34 +131,6 @@ jobs: working-directory: ./pdns/recursordist/ - run: inv ci-rec-configure - run: inv ci-rec-make-bear - - name: Normalize paths in compilation DB - working-directory: . - run: python3 .github/scripts/normalize_paths_in_compilation_database.py --version $BUILDER_VERSION ./pdns/recursordist/pdns-recursor-$BUILDER_VERSION/compile_commands.json - - name: Copy compilation DB - working-directory: . - run: cp ./pdns/recursordist/pdns-recursor-$BUILDER_VERSION/compile_commands.json . - - run: ln -s .clang-tidy.full .clang-tidy - working-directory: . - - name: Run clang-tidy - working-directory: . - run: git diff -U0 HEAD^..HEAD | python3 .github/scripts/git-filter.py | python3 .github/scripts/clang-tidy-diff.py -clang-tidy-binary /usr/bin/clang-tidy-${CLANG_VERSION} -extra-arg=-ferror-limit=0 -p1 -export-fixes clang-tidy-rec.yml - - name: Print clang-tidy fixes YAML - working-directory: . - shell: bash - run: | - if [ -f clang-tidy-rec.yml ]; then - cat clang-tidy-rec.yml - fi - - name: Result annotations - id: clang-tidy-annotations - working-directory: . - shell: bash - run: | - if [ -f clang-tidy-rec.yml ]; then - set +e - python .github/scripts/clang-tidy.py --fixes-file clang-tidy-rec.yml - echo "failed=$?" >> $GITHUB_OUTPUT - fi - run: inv ci-rec-run-unit-tests - run: inv generate-coverage-info ./testrunner $GITHUB_WORKSPACE if: ${{ matrix.sanitizers != 'tsan' }} @@ -232,8 +174,6 @@ jobs: defaults: run: working-directory: ./pdns/dnsdistdist/dnsdist-${{ env.BUILDER_VERSION }} - outputs: - clang-tidy-failed: ${{ steps.clang-tidy-annotations.outputs.failed }} steps: - uses: actions/checkout@v3 with: @@ -265,34 +205,6 @@ jobs: working-directory: ./pdns/dnsdistdist/ - run: inv ci-dnsdist-configure ${{ matrix.features }} - run: inv ci-dnsdist-make-bear - - name: Normalize paths in compilation DB - working-directory: . - run: python3 .github/scripts/normalize_paths_in_compilation_database.py --version $BUILDER_VERSION ./pdns/dnsdistdist/dnsdist-$BUILDER_VERSION/compile_commands.json - - name: Copy compilation DB - run: cp ./pdns/dnsdistdist/dnsdist-$BUILDER_VERSION/compile_commands.json compile_commands.json - working-directory: . - - run: ln -s .clang-tidy.full .clang-tidy - working-directory: . - - name: Run clang-tidy - working-directory: . - run: git diff -U0 HEAD^..HEAD | python3 .github/scripts/git-filter.py | python3 .github/scripts/clang-tidy-diff.py -clang-tidy-binary /usr/bin/clang-tidy-${CLANG_VERSION} -extra-arg=-ferror-limit=0 -p1 -export-fixes clang-tidy-dnsdist.yml - - name: Print clang-tidy fixes YAML - working-directory: . - shell: bash - run: | - if [ -f clang-tidy-dnsdist.yml ]; then - cat clang-tidy-dnsdist.yml - fi - - name: Result annotations - id: clang-tidy-annotations - working-directory: . - shell: bash - run: | - if [ -f clang-tidy-dnsdist.yml ]; then - set +e - python .github/scripts/clang-tidy.py --fixes-file clang-tidy-dnsdist.yml - echo "failed=$?" >> $GITHUB_OUTPUT - fi - run: inv ci-dnsdist-run-unit-tests - run: inv generate-coverage-info ./testrunner $GITHUB_WORKSPACE if: ${{ matrix.sanitizers != 'tsan' }} @@ -708,25 +620,6 @@ jobs: - run: inv install-swagger-tools - run: inv swagger-syntax-check - check-clang-tidy: - needs: [build-auth, build-dnsdist, build-recursor] - runs-on: ubuntu-20.04 - name: Check whether clang-tidy succeeded - steps: - - run: | - if [ "x${{ needs.build-auth.outputs.clang-tidy-failed }}" != "x" -a "${{ needs.build-auth.outputs.clang-tidy-failed }}" != "0" ]; then - echo "::error::Auth clang-tidy failed" - exit 1 - fi - if [ "x${{needs.build-recursor.outputs.clang-tidy-failed}}" != "x" -a "${{needs.build-recursor.outputs.clang-tidy-failed}}" != "0" ]; then - echo "::error::Rec clang-tidy failed" - exit 1 - fi - if [ "x${{ needs.build-dnsdist.outputs.clang-tidy-failed }}" != "x" -a "${{ needs.build-dnsdist.outputs.clang-tidy-failed }}" != "0" ]; then - echo "::error::dnsdist clang-tidy failed" - exit 1 - fi - collect: needs: - build-auth @@ -740,7 +633,6 @@ jobs: - test-recursor-api - test-recursor-regression - test-recursor-bulk - - check-clang-tidy if: success() || failure() runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b8c7047261..29561a858e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,4 +1,4 @@ -name: "CodeQL" +name: "CodeQL and clang-tidy" on: push: @@ -30,6 +30,21 @@ jobs: # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + env: + COMPILER: gcc + UNIT_TESTS: yes + FUZZING_TARGETS: yes + COVERAGE: no + OPTIMIZATIONS: no + # for clang-tidy only, not compilation + CLANG_VERSION: '13' + REPO_HOME: ${{ github.workspace }} + + outputs: + clang-tidy-annotations-auth: ${{ steps.clang-tidy-annotations-auth.outputs.failed }} + clang-tidy-annotations-dnsdist: ${{ steps.clang-tidy-annotations-dnsdist.outputs.failed }} + clang-tidy-annotations-rec: ${{ steps.clang-tidy-annotations-rec.outputs.failed }} + steps: - uses: PowerDNS/pdns/set-ubuntu-mirror@meta - name: Checkout repository @@ -53,7 +68,7 @@ jobs: # TODO: go through +security-and-quality (400 alerts) once, then see if we can upgrade to it # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. + # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main @@ -72,72 +87,163 @@ jobs: - name: Update repository metadata run: | sudo apt-get update - - name: Install dependencies - run: | - sudo apt-get -qq -y --no-install-recommends --allow-downgrades install \ - bison \ - default-libmysqlclient-dev \ - flex \ - libboost-all-dev \ - libcap-dev \ - libcdb-dev \ - libcurl4-openssl-dev \ - libedit-dev \ - libfstrm-dev \ - libgeoip-dev \ - libgnutls28-dev \ - libh2o-evloop-dev \ - libkrb5-dev \ - libldap2-dev \ - liblmdb-dev \ - liblua5.3-dev \ - libmaxminddb-dev \ - libnghttp2-dev \ - libp11-kit-dev \ - libpq-dev \ - libre2-dev \ - libsnmp-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libwslay-dev \ - libyaml-cpp-dev \ - ragel \ - rustc \ - unixodbc-dev + - name: Update repository metadata + run: | + sudo apt-get -qq -y --no-install-recommends install python3-pip python3-invoke + + - name: Install clang-tidy tools + run: | + inv install-clang-tidy-tools + + - name: Install dependencies for auth + if: matrix.product == 'auth' + run: | + inv install-auth-build-deps + - name: Autoreconf auth + if: matrix.product == 'auth' + run: | + inv ci-autoconf + - name: Configure auth + if: matrix.product == 'auth' + run: | + inv ci-auth-configure - name: Build auth if: matrix.product == 'auth' run: | - autoreconf -vfi - ./configure --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' --enable-tools --enable-ixfrdist --enable-dns-over-tls --enable-experimental-pkcs11 --with-libsodium --enable-lua-records CFLAGS='-O0' CXXFLAGS='-O0' - make -j8 -C ext - make -j8 -C modules - make -j8 -C pdns + inv ci-auth-make-bear + - run: ln -s .clang-tidy.full .clang-tidy + - name: Run clang-tidy for auth + if: matrix.product == 'auth' + run: git diff -U0 HEAD^..HEAD | python3 .github/scripts/git-filter.py | clang-tidy-diff -clang-tidy-binary /usr/bin/clang-tidy-${CLANG_VERSION} -extra-arg=-ferror-limit=0 -p1 -export-fixes clang-tidy-auth.yml + - name: Print clang-tidy fixes YAML for auth + if: matrix.product == 'auth' + shell: bash + run: | + if [ -f clang-tidy-auth.yml ]; then + cat clang-tidy-auth.yml + fi + - name: Result annotations for auth + if: matrix.product == 'auth' + id: clang-tidy-annotations-auth + shell: bash + run: | + if [ -f clang-tidy-auth.yml ]; then + set +e + python3 .github/scripts/clang-tidy.py --fixes-file clang-tidy-auth.yml + echo "failed=$?" >> $GITHUB_OUTPUT + fi + - name: Install dependencies for dnsdist + if: matrix.product == 'dnsdist' + run: | + inv install-dnsdist-build-deps + - name: Autoreconf dnsdist + if: matrix.product == 'dnsdist' + working-directory: ./pdns/dnsdistdist/ + run: | + inv ci-autoconf + - name: Configure dnsdist + if: matrix.product == 'dnsdist' + working-directory: ./pdns/dnsdistdist/ + run: | + inv ci-dnsdist-configure full - name: Build dnsdist if: matrix.product == 'dnsdist' + working-directory: ./pdns/dnsdistdist/ + run: | + inv ci-dnsdist-make-bear + - run: ln -s ../../.clang-tidy.full .clang-tidy + working-directory: ./pdns/dnsdistdist/ + - name: Run clang-tidy for dnsdist + if: matrix.product == 'dnsdist' + working-directory: ./pdns/dnsdistdist/ + run: git diff -U0 HEAD^..HEAD | python3 ../../.github/scripts/git-filter.py | clang-tidy-diff -clang-tidy-binary /usr/bin/clang-tidy-${CLANG_VERSION} -extra-arg=-ferror-limit=0 -p1 -export-fixes clang-tidy-dnsdist.yml + - name: Print clang-tidy fixes YAML for dnsdist + if: matrix.product == 'dnsdist' + working-directory: ./pdns/dnsdistdist/ + shell: bash run: | - cd pdns/dnsdistdist - autoreconf -vfi - ./configure --enable-unit-tests --enable-dnstap --enable-dnscrypt --enable-dns-over-tls --enable-dns-over-https --with-h2o LIBS=-lwslay CFLAGS='-O0' CXXFLAGS='-O0' - make -j8 -C ext/arc4random - make -j8 -C ext/ipcrypt - make -j8 -C ext/yahttp - make -j4 dnsdist + if [ -f clang-tidy-dnsdist.yml ]; then + cat clang-tidy-dnsdist.yml + fi + - name: Result annotations for dnsdist + if: matrix.product == 'dnsdist' + working-directory: ./pdns/dnsdistdist/ + id: clang-tidy-annotations-dnsdist + shell: bash + run: | + if [ -f clang-tidy-dnsdist.yml ]; then + set +e + python3 .github/scripts/clang-tidy.py --fixes-file clang-tidy-dnsdist.yml + echo "failed=$?" >> $GITHUB_OUTPUT + fi - - name: Build recursor + - name: Install dependencies for rec + if: matrix.product == 'rec' + run: | + inv install-rec-build-deps + - run: inv ci-install-rust ${{ env.REPO_HOME }} + working-directory: ./pdns/recursordist/ + - name: Autoreconf rec + if: matrix.product == 'rec' + working-directory: ./pdns/recursordist/ + run: | + inv ci-autoconf + - name: Configure rec if: matrix.product == 'rec' + working-directory: ./pdns/recursordist/ run: | - cd pdns/recursordist - autoreconf -vfi - ./configure --enable-unit-tests --enable-nod --enable-dnstap CFLAGS='-O0' CXXFLAGS='-O0' - make -j8 -C ext - make -j8 -C settings - make -j8 -C settings/rust - make htmlfiles.h - make -j4 pdns_recursor rec_control + inv ci-rec-configure + - name: Build rec + if: matrix.product == 'rec' + working-directory: ./pdns/recursordist/ + run: | + inv ci-rec-make-bear + - run: ln -s ../../.clang-tidy.full .clang-tidy + working-directory: ./pdns/recursordist/ + - name: Run clang-tidy for rec + if: matrix.product == 'rec' + working-directory: ./pdns/recursordist/ + run: git diff -U0 HEAD^..HEAD | python3 ../../.github/scripts/git-filter.py | clang-tidy-diff -clang-tidy-binary /usr/bin/clang-tidy-${CLANG_VERSION} -extra-arg=-ferror-limit=0 -p1 -export-fixes clang-tidy-rec.yml + - name: Print clang-tidy fixes YAML for rec + if: matrix.product == 'rec' + working-directory: ./pdns/recursordist/ + shell: bash + run: | + if [ -f clang-tidy-rec.yml ]; then + cat clang-tidy-rec.yml + fi + - name: Result annotations for rec + if: matrix.product == 'rec' + working-directory: ./pdns/recursordist/ + id: clang-tidy-annotations-rec + shell: bash + run: | + if [ -f clang-tidy-rec.yml ]; then + set +e + python3 .github/scripts/clang-tidy.py --fixes-file clang-tidy-rec.yml + echo "failed=$?" >> $GITHUB_OUTPUT + fi - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 + + check-clang-tidy: + needs: analyze + runs-on: ubuntu-20.04 + name: Check whether clang-tidy succeeded + steps: + - run: | + if [ "x${{ needs.analyze.outputs.clang-tidy-annotations-auth }}" != "x" -a "${{ needs.analyze.outputs.clang-tidy-annotations-auth }}" != "0" ]; then + echo "::error::Auth clang-tidy failed" + exit 1 + fi + if [ "x${{ needs.analyze.outputs.clang-tidy-annotations-dnsdist }}" != "x" -a "${{ needs.analyze.outputs.clang-tidy-annotations-dnsdist }}" != "0" ]; then + echo "::error::DNSdist clang-tidy failed" + exit 1 + fi + if [ "x${{needs.analyze.outputs.clang-tidy-annotations-rec }}" != "x" -a "${{needs.analyze.outputs.clang-tidy-annotations-rec }}" != "0" ]; then + echo "::error::Rec clang-tidy failed" + exit 1 + fi diff --git a/tasks.py b/tasks.py index e647349c1f..80de49e97e 100644 --- a/tasks.py +++ b/tasks.py @@ -180,7 +180,7 @@ def install_libdecaf(c, product): c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf') with c.cd('/tmp/libdecaf'): c.run('git checkout 41f349') - c.run(f'CC=clang-{clang_version} CXX=clang-{clang_version} ' + c.run(f'CC={get_c_compiler()} CXX={get_cxx_compiler()} ' 'cmake -B build ' '-DCMAKE_INSTALL_PREFIX=/usr/local ' '-DCMAKE_INSTALL_LIBDIR=lib ' @@ -373,10 +373,24 @@ def get_sanitizers(): sanitizers = ' '.join(sanitizers) return sanitizers +def get_c_compiler(): + if os.getenv('COMPILER', 'clang') == 'clang': + return f'clang-{clang_version}' + return 'gcc' + +def get_cxx_compiler(): + if os.getenv('COMPILER', 'clang') == 'clang': + return f'clang++-{clang_version}' + return 'g++' + +def get_optimizations(): + if os.getenv('OPTIMIZATIONS', 'yes') == 'yes': + return "-O1" + return "-O0" def get_cflags(): return " ".join([ - "-O1", + get_optimizations(), "-Werror=vla", "-Werror=shadow", "-Wformat=2", @@ -397,8 +411,8 @@ def get_base_configure_cmd(): f'CFLAGS="{get_cflags()}"', f'CXXFLAGS="{get_cxxflags()}"', './configure', - f"CC='clang-{clang_version}'", - f"CXX='clang++-{clang_version}'", + f"CC='{get_c_compiler()}'", + f"CXX='{get_cxx_compiler()}'", "--enable-option-checking=fatal", "--enable-systemd", "--with-libsodium", @@ -440,6 +454,7 @@ def ci_auth_configure(c): "LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib'", f"--with-modules='{modules}'", "--enable-tools", + "--enable-dns-over-tls", "--enable-experimental-pkcs11", "--enable-experimental-gss-tsig", "--enable-remotebackend-zeromq", @@ -555,18 +570,18 @@ def ci_dnsdist_configure(c, features): coverage = '--enable-coverage=clang' if is_coverage_enabled() else '' cflags = get_cflags() cxxflags = " ".join([get_cxxflags(), additional_flags]) + tools = f'''AR=llvm-ar-{clang_version} RANLIB=llvm-ranlib-{clang_version}''' if os.getenv('COMPILER', 'clang') == 'clang' else '' res = c.run(f'''CFLAGS="%s" \ CXXFLAGS="%s" \ - AR=llvm-ar-{clang_version} \ - RANLIB=llvm-ranlib-{clang_version} \ + %s \ ./configure \ - CC='clang-{clang_version}' \ - CXX='clang++-{clang_version}' \ + CC='{get_c_compiler()}' \ + CXX='{get_cxx_compiler()}' \ --enable-option-checking=fatal \ --enable-fortify-source=auto \ --enable-auto-var-init=pattern \ --enable-lto=thin \ - --prefix=/opt/dnsdist %s %s %s %s %s''' % (cflags, cxxflags, features_set, sanitizers, unittests, fuzztargets, coverage), warn=True) + --prefix=/opt/dnsdist %s %s %s %s %s''' % (cflags, cxxflags, tools, features_set, sanitizers, unittests, fuzztargets, coverage), warn=True) if res.exited != 0: c.run('cat config.log') raise UnexpectedExit(res)