From 62515e8e9d750f02e70545905d16fc025357893d Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Sat, 14 Dec 2024 13:29:10 +0100 Subject: [PATCH] scripts/mdlinkcheck: markdown link checker This script parses all markdown files in the repository, extracts all links and verifies that they work. It makes sure to only check the URLs once, even if used in multiple links. There is a whitelist for URLs we deem unnecessary to check. It uses curl to do the checks. As a bonus, this makes it easy to run this check locally. Closes #15742 --- .github/workflows/checkdocs.yml | 12 +-- scripts/Makefile.am | 2 +- scripts/mdlinkcheck | 165 ++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 11 deletions(-) create mode 100755 scripts/mdlinkcheck diff --git a/.github/workflows/checkdocs.yml b/.github/workflows/checkdocs.yml index 0f19157625..8f74f463f8 100644 --- a/.github/workflows/checkdocs.yml +++ b/.github/workflows/checkdocs.yml @@ -85,22 +85,14 @@ jobs: # - name: check special prose # run: proselint docs/internals/CHECKSRC.md docs/libcurl/curl_mprintf.md docs/libcurl/opts/CURLOPT_INTERFACE.md docs/cmdline-opts/interface.md - # Docs: https://github.com/UmbrellaDocs/action-linkspector linkcheck: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 name: checkout - - name: trim the cmdline docs markdown files - run: find docs/cmdline-opts -name "*.md" ! -name "_*" ! -name MANPAGE.md -print0 | xargs -0 -n1 .github/scripts/cleancmd.pl - - - name: Run linkspector - uses: umbrelladocs/action-linkspector@fc382e19892aca958e189954912fe379a8df270c # v1 - with: - github_token: ${{ secrets.github_token }} - reporter: github-pr-review - fail_on_error: true + - name: Run mdlinkcheck + run: ./scripts/mdlinkcheck spellcheck: runs-on: ubuntu-latest diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 4454d42cf2..b29c99d892 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -24,7 +24,7 @@ EXTRA_DIST = coverage.sh completion.pl firefox-db2pem.sh checksrc.pl \ mk-ca-bundle.pl mk-unity.pl schemetable.c cd2nroff nroff2cd cdall cd2cd managen \ - dmaketgz maketgz release-tools.sh verify-release cmakelint.sh + dmaketgz maketgz release-tools.sh verify-release cmakelint.sh mdlinkcheck ZSH_FUNCTIONS_DIR = @ZSH_FUNCTIONS_DIR@ FISH_FUNCTIONS_DIR = @FISH_FUNCTIONS_DIR@ diff --git a/scripts/mdlinkcheck b/scripts/mdlinkcheck new file mode 100755 index 0000000000..395fc85cbd --- /dev/null +++ b/scripts/mdlinkcheck @@ -0,0 +1,165 @@ +#!/usr/bin/env perl +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### + +my %whitelist = ( + 'https://curl.se/' => 1, + 'https://curl.se/changes.html' => 1, + 'https://curl.se/dev/advisory.html' => 1, + 'https://curl.se/dev/builds.html' => 1, + 'https://curl.se/dev/code-style.html' => 1, + 'https://curl.se/dev/contribute.html' => 1, + 'https://curl.se/dev/internals.html' => 1, + 'https://curl.se/dev/secprocess.html' => 1, + 'https://curl.se/dev/sourceactivity.html' => 1, + 'https://curl.se/docs/' => 1, + 'https://curl.se/docs/bugbounty.html' => 1, + 'https://curl.se/docs/caextract.html' => 1, + 'https://curl.se/docs/copyright.html' => 1, + 'https://curl.se/docs/install.html' => 1, + 'https://curl.se/docs/knownbugs.html' => 1, + 'https://curl.se/docs/manpage.html' => 1, + 'https://curl.se/docs/security.html' => 1, + 'https://curl.se/docs/sslcerts.html' => 1, + 'https://curl.se/docs/thanks.html' => 1, + 'https://curl.se/docs/todo.html' => 1, + 'https://curl.se/docs/vulnerabilities.html' => 1, + 'https://curl.se/libcurl/' => 1, + 'https://curl.se/libcurl/c/CURLOPT_SSLVERSION.html' => 1, + 'https://curl.se/libcurl/c/CURLOPT_SSL_CIPHER_LIST.html' => 1, + 'https://curl.se/libcurl/c/CURLOPT_TLS13_CIPHERS.html' => 1, + 'https://curl.se/libcurl/c/libcurl.html' => 1, + 'https://curl.se/logo/curl-logo.svg' => 1, + 'https://curl.se/mail/' => 1, + 'https://curl.se/mail/etiquette.html' => 1, + 'https://curl.se/mail/list.cgi?list=curl-distros' => 1, + 'https://curl.se/mail/list.cgi?list=curl-library' => 1, + 'https://curl.se/rfc/cookie_spec.html' => 1, + 'https://curl.se/rfc/rfc2255.txt' => 1, + 'https://curl.se/sponsors.html' => 1, + 'https://curl.se/support.html' => 1, + + 'https://github.com/curl/curl' => 1, + 'https://github.com/curl/curl-fuzzer' => 1, + 'https://github.com/curl/curl-www' => 1, + 'https://github.com/curl/curl/discussions' => 1, + 'https://github.com/curl/curl/issues' => 1, + 'https://github.com/curl/curl/labels/help%20wanted' => 1, + 'https://github.com/curl/curl/pulls' => 1, + + ); + +# list all .md files in the repo +my @files=`git ls-files '**.md'`; + +sub storelink { + my ($f, $line, $link) = @_; + my $o = $link; + + if($link =~ /^\#/) { + # ignore local-only links + return; + } + # cut off any anchor + $link =~ s:\#.*\z::; + + if($link =~ /^(https|http):/) { + $url{$link} .= "$f:$line "; + return; + } + + # a file link + my $dir = $f; + $dir =~ s:([^/]*\z)::; + + while($link =~ s:^\.\.\/::) { + $dir =~ s:^([^/]*)/::; + } + + $flink{"./$dir$link"} .= "$f:$line "; +} + +sub findlinks { + my ($f) = @_; + my $line = 1; + open(F, "<:crlf", "$f") || + return; + + while() { + if(/\]\(([^)]*)/) { + my $link = $1; + #print "$f:$line $link\n"; + storelink($f, $line, $link); + } + $line++; + } + close(F); +} + +sub checkurl { + my ($url) = @_; + + if($whitelist{$url}) { + #print "$url is whitelisted\n"; + return 0; + } + + print "check $url\n"; + my $curlcmd="curl -ILfsm10 --retry 2 --retry-delay 5 -A \"Mozilla/curl.se link-probe\""; + my @content = `$curlcmd \"$url\"`; + if(!$content[0]) { + print STDERR "FAIL\n"; + return 1; # fail + } + return 0; # ok +} + +for my $f (@files) { + chomp $f; + findlinks($f); +} + +my $error; + +for my $u (sort keys %url) { + my $r = checkurl($u); + + if($r) { + for my $f (split(/ /, $url{$l})) { + printf "%s ERROR links to missing URL %s\n", $f, $u; + $error++; + } + } +} + +for my $l (sort keys %flink) { + if(! -r $l) { + for my $f (split(/ /, $flink{$l})) { + printf "%s ERROR links to missing file %s\n", $f, $l; + $error++; + } + } +} + +exit 1 if ($error); -- 2.47.3