From 5629bb7cf62903ad6afdb0cf146b168cd88a047a Mon Sep 17 00:00:00 2001 From: Dan Fandrich Date: Thu, 22 Aug 2024 21:24:03 -0700 Subject: [PATCH] CI: consolidate workflows for source and docs check A bunch of tiny jobs that run various source or documentation checks are consolidated into two workflow files: checksrc.yml and checkdocs.yml. This reduces the proliferation of new files containing one-line checks and brings those that operate similarly together for better reasoning about them. The man-examples check is also now running again for the first time in 7 months. Various calls to find, xargs and git ls-files are changed where possible to use NUL line terminators in pipes to avoid issues with oddly-named files that might find their way into the repo. Closes #14654 --- .github/labeler.yml | 4 +- .github/workflows/badwords.yml | 42 --------- .github/workflows/binarycheck.yml | 23 ----- .github/workflows/checkdocs.yml | 146 +++++++++++++++++++++++++++++ .github/workflows/checksrc.yml | 77 +++++++++++++-- .github/workflows/codespell.yml | 39 -------- .github/workflows/linkcheck.yml | 41 -------- .github/workflows/man-examples.yml | 37 -------- .github/workflows/proselint.yml | 74 --------------- .github/workflows/reuse.yml | 29 ------ .github/workflows/shellcheck.yml | 28 ------ .github/workflows/spacecheck.yml | 28 ------ .github/workflows/spellcheck.yml | 50 ---------- .github/workflows/synopsis.yml | 34 ------- 14 files changed, 218 insertions(+), 434 deletions(-) delete mode 100644 .github/workflows/badwords.yml delete mode 100644 .github/workflows/binarycheck.yml create mode 100644 .github/workflows/checkdocs.yml delete mode 100644 .github/workflows/codespell.yml delete mode 100644 .github/workflows/linkcheck.yml delete mode 100644 .github/workflows/man-examples.yml delete mode 100644 .github/workflows/proselint.yml delete mode 100644 .github/workflows/reuse.yml delete mode 100644 .github/workflows/shellcheck.yml delete mode 100644 .github/workflows/spacecheck.yml delete mode 100644 .github/workflows/spellcheck.yml delete mode 100644 .github/workflows/synopsis.yml diff --git a/.github/labeler.yml b/.github/labeler.yml index 5a5029108d..e6a64e873d 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -185,9 +185,7 @@ documentation: - all: - changed-files: - any-glob-to-all-files: "{\ - .github/workflows/badwords.yml,\ - .github/workflows/man-examples.yml,\ - .github/workflows/synopsis.yml,\ + .github/workflows/checkdocs.yml,\ .github/scripts/badwords.*,\ .github/scripts/cd2cd,\ .github/scripts/cd2nroff,\ diff --git a/.github/workflows/badwords.yml b/.github/workflows/badwords.yml deleted file mode 100644 index dbdbb13f95..0000000000 --- a/.github/workflows/badwords.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (C) Daniel Stenberg, , et al. -# -# SPDX-License-Identifier: curl - -name: badwords - -on: - # Trigger the workflow on push or pull requests, but only for the - # master branch - push: - branches: - - master - - '*/ci' - pull_request: - branches: - - master - -permissions: {} - -jobs: - docs: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: check - run: ./.github/scripts/badwords.pl < .github/scripts/badwords.txt docs/*.md docs/libcurl/*.md docs/libcurl/opts/*.md docs/cmdline-opts/*.md docs/TODO docs/KNOWN_BUGS tests/*.md - - source: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - # we allow some extra in source code - - name: trim wordlist - run: grep -Ev '(\\bwill| url | dir )' .github/scripts/badwords.txt > .github/scripts/source.txt - - - name: check - run: ./.github/scripts/badwords.pl < .github/scripts/source.txt `git ls-files -- src lib include` diff --git a/.github/workflows/binarycheck.yml b/.github/workflows/binarycheck.yml deleted file mode 100644 index e3f4c20412..0000000000 --- a/.github/workflows/binarycheck.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Daniel Stenberg, , et al. -# -# SPDX-License-Identifier: curl - -name: bincheck -on: - push: - branches: - - master - pull_request: - branches: - - master - -permissions: {} - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Check for binary files - run: ./.github/scripts/binarycheck.pl diff --git a/.github/workflows/checkdocs.yml b/.github/workflows/checkdocs.yml new file mode 100644 index 0000000000..60b8d1cee5 --- /dev/null +++ b/.github/workflows/checkdocs.yml @@ -0,0 +1,146 @@ +# Copyright (C) Daniel Stenberg, , et al. +# +# SPDX-License-Identifier: curl + +# This workflow contains tests that operate on documentation files only. Some +# checks modify the source so they cannot be combined into a single job. + +name: Docs + +on: + push: + branches: + - master + - '*/ci' + paths: + - '.github/workflows/checkdocs.yml' + - '.github/scripts/**' + - '**.md' + - 'docs/*' + pull_request: + branches: + - master + paths: + - '.github/workflows/checkdocs.yml' + - '.github/scripts/**' + - '**.md' + - 'docs/*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +permissions: {} + +jobs: + proselint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + name: checkout + + - name: install prereqs + run: | + sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list + sudo apt-get install python3-proselint + + # config file help: https://github.com/amperser/proselint/ + - name: create proselint config + run: | + cat < $HOME/.proselintrc + { + "checks": { + "typography.diacritical_marks": false, + "typography.symbols": false, + "annotations.misc": false, + "security.password": false + } + } + JSON + + - name: trim headers off all *.md files + run: git ls-files -z '*.md' | xargs -0 -n1 .github/scripts/trimmarkdownheader.pl + + - name: check prose + run: git ls-files -z '*.md' | grep -Evz 'CHECKSRC.md|DISTROS.md|CURLOPT_INTERFACE.md|interface.md' | xargs -0 proselint README + + # This is for CHECKSRC and files with aggressive exclamation mark needs + - name: create second proselint config + run: | + cat < $HOME/.proselintrc + { + "checks": { + "typography.diacritical_marks": false, + "typography.symbols": false, + "typography.exclamation": false, + "annotations.misc": false + } + } + JSON + + - name: check special prose + run: proselint docs/CHECKSRC.md docs/libcurl/opts/CURLOPT_INTERFACE.md docs/cmdline-opts/interface.md + + # Docs: https://github.com/marketplace/actions/markdown-link-check + linkcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + name: checkout + + - name: trim the cmdline docs markdown files + run: find docs/cmdline-opts -name "*.md" ! -name "_*" ! -name MANPAGE.md -print0 | xargs -0 -n1 .github/scripts/cleancmd.pl + + - uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1 + with: + use-quiet-mode: 'yes' + + spellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + name: checkout + + - name: trim all man page *.md files + run: find docs -name "*.md" ! -name "_*" -print0 | xargs -0 -n1 .github/scripts/cleancmd.pl + + - name: trim libcurl man page *.md files + run: find docs/libcurl \( -name "curl_*.md" -o -name "libcurl*.md" \) -print0 | xargs -0 -n1 .github/scripts/cleanspell.pl + + - name: trim libcurl option man page *.md files + run: find docs/libcurl/opts -name "CURL*.md" -print0 | xargs -0 -n1 .github/scripts/cleanspell.pl + + - name: trim cmdline docs markdown _*.md files + run: find docs/cmdline-opts -name "_*.md" -print0 | xargs -0 -n1 .github/scripts/cleancmd.pl --no-header + + - name: setup the custom wordlist + run: grep -v '^#' .github/scripts/spellcheck.words > wordlist.txt + + - name: Check Spelling + uses: rojopolis/spellcheck-github-actions@a0fba0ca8b9e552d4241ea5ccfaa4ca4162622d0 # v0 + with: + config_path: .github/scripts/spellcheck.yaml + + badwords-synopsis: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + name: checkout + + - name: badwords + run: .github/scripts/badwords.pl < .github/scripts/badwords.txt docs/*.md docs/libcurl/*.md docs/libcurl/opts/*.md docs/cmdline-opts/*.md docs/TODO docs/KNOWN_BUGS tests/*.md + + - name: verify-synopsis + run: .github/scripts/verify-synopsis.pl docs/libcurl/curl*.md + + man-examples: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + name: checkout + + - name: render nroff versions + run: autoreconf -fi && ./configure --without-ssl --without-libpsl && make -C docs + + - name: verify examples + run: .github/scripts/verify-examples.pl docs/libcurl/curl*.3 docs/libcurl/opts/*.3 diff --git a/.github/workflows/checksrc.yml b/.github/workflows/checksrc.yml index 0115bc5119..5992af2fe6 100644 --- a/.github/workflows/checksrc.yml +++ b/.github/workflows/checksrc.yml @@ -2,28 +2,93 @@ # # SPDX-License-Identifier: curl -name: checksrc +# This workflow contains checks at the source code level only. + +name: Source on: - # Trigger the workflow on push or pull requests, but only for the - # master branch push: branches: - master - '*/ci' + paths-ignore: + - '**/*.md' + - '**/CMakeLists.txt' + - '.azure-pipelines.yml' + - '.circleci/**' + - 'appveyor.*' + - 'CMake/**' + - 'plan9/**' + - 'tests/data/**' + - 'winbuild/**' pull_request: branches: - master + paths-ignore: + - '**/*.md' + - '**/CMakeLists.txt' + - '.azure-pipelines.yml' + - '.circleci/**' + - 'appveyor.*' + - 'CMake/**' + - 'plan9/**' + - 'tests/data/**' + - 'winbuild/**' permissions: {} jobs: - all: + checksrc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + name: checkout + + - name: check + run: git ls-files -z "*.[ch]" | xargs -0 -n1 ./scripts/checksrc.pl + + codespell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + name: checkout + - name: install + run: | + sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list + sudo apt-get update + sudo apt-get install codespell + + - name: spellcheck + run: codespell --skip src/tool_hugehelp.c -I .github/scripts/codespell-ignore.txt include src lib + + reuse: runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + name: checkout + + - name: REUSE Compliance Check + uses: fsfe/reuse-action@3ae3c6bdf1257ab19397fab11fd3312144692083 # v4 + miscchecks: + runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + name: checkout - - name: check - run: git ls-files "*.[ch]" | xargs -n1 ./scripts/checksrc.pl + - name: shellcheck + run: .github/scripts/shellcheck.sh + + - name: spacecheck + run: .github/scripts/spacecheck.pl + + - name: binarycheck + run: .github/scripts/binarycheck.pl + + # we allow some extra in source code + - name: badwords + run: > + grep -Ev '(\\bwill| url | dir )' .github/scripts/badwords.txt | + .github/scripts/badwords.pl `git ls-files -- src lib include` diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml deleted file mode 100644 index 4aa8cda89d..0000000000 --- a/.github/workflows/codespell.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) Daniel Stenberg, , et al. -# -# SPDX-License-Identifier: curl - -name: Codespell - -on: - push: - branches: - - master - - '*/ci' - paths: - - 'lib/**' - - 'src/**' - - 'include/**' - pull_request: - branches: - - master - - 'lib/**' - - 'src/**' - - 'include/**' - -permissions: {} - -jobs: - codespell: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: install - run: | - sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - sudo apt-get update - sudo apt-get install codespell - - - name: spellcheck - run: codespell --skip src/tool_hugehelp.c -I .github/scripts/codespell-ignore.txt include src lib diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml deleted file mode 100644 index fff8515d4b..0000000000 --- a/.github/workflows/linkcheck.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (C) Daniel Stenberg, , et al. -# -# SPDX-License-Identifier: curl - -name: Markdown links - -on: - push: - branches: - - master - - '*/ci' - paths: - - '.github/workflows/linkcheck.yml' - - '**.md' - pull_request: - branches: - - master - paths: - - '.github/workflows/linkcheck.yml' - - '**.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -permissions: {} - -jobs: - # Docs: https://github.com/marketplace/actions/markdown-link-check - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: checkout - - - name: trim the cmdline docs markdown files - run: find docs/cmdline-opts -name "*.md" ! -name "_*" ! -name MANPAGE.md | xargs -n1 ./.github/scripts/cleancmd.pl - - - uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1 - with: - use-quiet-mode: 'yes' diff --git a/.github/workflows/man-examples.yml b/.github/workflows/man-examples.yml deleted file mode 100644 index 502d590bcb..0000000000 --- a/.github/workflows/man-examples.yml +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) Daniel Stenberg, , et al. -# -# SPDX-License-Identifier: curl - -name: manpage examples - -on: - push: - branches: - - master - - '*/ci' - paths: - - 'docs/libcurl/curl_*.3' - - 'docs/libcurl/opts/*.3' - - '.github/scripts/verify-examples.pl' - pull_request: - branches: - - master - paths: - - 'docs/libcurl/curl_*.3' - - 'docs/libcurl/opts/*.3' - - '.github/scripts/verify-examples.pl' - -permissions: {} - -jobs: - verify: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: render nroff versions - run: autoreconf -fi && ./configure --without-ssl --without-libpsl && make -C docs - - - name: verify examples - run: ./.github/scripts/verify-examples.pl docs/libcurl/curl*.3 docs/libcurl/opts/*.3 diff --git a/.github/workflows/proselint.yml b/.github/workflows/proselint.yml deleted file mode 100644 index 54075ddf6d..0000000000 --- a/.github/workflows/proselint.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (C) Daniel Stenberg, , et al. -# -# SPDX-License-Identifier: curl - -name: proselint - -on: - push: - branches: - - master - - '*/ci' - paths: - - '.github/workflows/proselint.yml' - - '**.md' - pull_request: - branches: - - master - paths: - - '.github/workflows/proselint.yml' - - '**.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -permissions: {} - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: install prereqs - run: | - sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - sudo apt-get install python3-proselint - - # config file help: https://github.com/amperser/proselint/ - - name: create proselint config - run: | - cat < $HOME/.proselintrc - { - "checks": { - "typography.diacritical_marks": false, - "typography.symbols": false, - "annotations.misc": false, - "security.password": false - } - } - JSON - - - name: trim headers off all *.md files - run: git ls-files '*.md' | xargs -n1 ./.github/scripts/trimmarkdownheader.pl - - - name: check prose - run: git ls-files '*.md' | grep -Ev 'CHECKSRC.md|DISTROS.md|CURLOPT_INTERFACE.md|interface.md' | xargs proselint README - - # This is for CHECKSRC and files with aggressive exclamation mark needs - - name: create second proselint config - run: | - cat < $HOME/.proselintrc - { - "checks": { - "typography.diacritical_marks": false, - "typography.symbols": false, - "typography.exclamation": false, - "annotations.misc": false - } - } - JSON - - - name: check special prose - run: proselint docs/CHECKSRC.md docs/libcurl/opts/CURLOPT_INTERFACE.md docs/cmdline-opts/interface.md diff --git a/.github/workflows/reuse.yml b/.github/workflows/reuse.yml deleted file mode 100644 index fc2371fc4e..0000000000 --- a/.github/workflows/reuse.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) Daniel Stenberg, , et al. -# SPDX-FileCopyrightText: 2022 Free Software Foundation Europe e.V. -# -# SPDX-License-Identifier: curl - -name: REUSE compliance - -on: - push: - branches: - - master - - '*/ci' - pull_request: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -permissions: {} - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - name: REUSE Compliance Check - uses: fsfe/reuse-action@3ae3c6bdf1257ab19397fab11fd3312144692083 # v4 diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml deleted file mode 100644 index 524a828482..0000000000 --- a/.github/workflows/shellcheck.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (C) Viktor Szakats -# -# SPDX-License-Identifier: curl - -name: shellcheck - -on: - push: - branches: - - master - pull_request: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -permissions: {} - -jobs: - shellcheck: - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - name: 'shellcheck' - run: .github/scripts/shellcheck.sh diff --git a/.github/workflows/spacecheck.yml b/.github/workflows/spacecheck.yml deleted file mode 100644 index 1a68185f98..0000000000 --- a/.github/workflows/spacecheck.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (C) Viktor Szakats -# -# SPDX-License-Identifier: curl - -name: spacecheck - -on: - push: - branches: - - master - pull_request: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -permissions: {} - -jobs: - spacecheck: - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - name: 'spacecheck' - run: .github/scripts/spacecheck.pl diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml deleted file mode 100644 index fd847324c9..0000000000 --- a/.github/workflows/spellcheck.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) Daniel Stenberg, , et al. -# -# SPDX-License-Identifier: curl - -name: spell -on: - push: - branches: - - master - paths: - - '**.md' - - '**/spellcheck.yml' - - '**/spellcheck.yaml' - - '.github/scripts/*' - pull_request: - branches: - - master - paths: - - '**.md' - - '**/spellcheck.yml' - - '**/spellcheck.yaml' - - '.github/scripts/*' - -permissions: {} - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: trim all man page *.md files - run: find docs -name "*.md" ! -name "_*" | xargs -n1 ./.github/scripts/cleancmd.pl - - - name: trim libcurl man page *.md files - run: find docs/libcurl -name "curl_*.md" -o -name "libcurl*.md" | xargs -n1 ./.github/scripts/cleanspell.pl - - - name: trim libcurl option man page *.md files - run: find docs/libcurl/opts -name "CURL*.md" | xargs -n1 ./.github/scripts/cleanspell.pl - - - name: trim cmdline docs markdown _*.md files - run: find docs/cmdline-opts -name "_*.md" | xargs -n1 ./.github/scripts/cleancmd.pl --no-header - - - name: setup the custom wordlist - run: grep -v '^#' .github/scripts/spellcheck.words > wordlist.txt - - - name: Check Spelling - uses: rojopolis/spellcheck-github-actions@a0fba0ca8b9e552d4241ea5ccfaa4ca4162622d0 # v0 - with: - config_path: .github/scripts/spellcheck.yaml diff --git a/.github/workflows/synopsis.yml b/.github/workflows/synopsis.yml deleted file mode 100644 index 10e7949b62..0000000000 --- a/.github/workflows/synopsis.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (C) Daniel Stenberg, , et al. -# -# SPDX-License-Identifier: curl - -name: SYNOPSIS - -on: - push: - branches: - - master - - '*/ci' - paths: - - 'docs/libcurl/curl_*.md' - - 'scripts/*.pl' - - '.github/workflows/*.yml' - pull_request: - branches: - - master - paths: - - 'docs/libcurl/curl_*.md' - - 'scripts/*.pl' - - '.github/workflows/*.yml' - -permissions: {} - -jobs: - verify: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: verify-synopsis - run: ./.github/scripts/verify-synopsis.pl docs/libcurl/curl*.md -- 2.47.3