CI fuzzing: Interactive access to fuzzer CI env using debug-fuzzer-* branch (#4416)

This makes it easy to use the same CI environment that runs the scheduled
fuzzing to execute reproducers, e.g. for debugging when a reproducer isn't
failing locally.

  * Push to debug-fuzzer-<protocol>, e.g. debug-fuzzer-radius
  * Watch the CI run in GitHub Actions until the tmate session starts
  * SSH to the emitted rendezvous point
  * Run scripts/build/fuzzer-fetch-artifacts, and follow the output

diff --git a/.github/workflows/ci-deb.yml b/.github/workflows/ci-deb.yml
index 4ce3b9838b35b4ab1ca3dec9560f7e0169940c77..3845a3145dee13b10bc23e992f4768234a07c0fd 100644 (file)
--- a/.github/workflows/ci-deb.yml
+++ b/.github/workflows/ci-deb.yml
@@ -5,6 +5,7 @@ on:
     branches-ignore:
       - coverity_scan
       - run-fuzzer**
+      - debug-fuzzer-**
   schedule:
     - cron: '0 20 * * *'
 

diff --git a/.github/workflows/ci-freebsd.yml b/.github/workflows/ci-freebsd.yml
index 71b7c79fc21a431c34afb80917228b2ad7422527..dfb97f2b0f4e9046db2e14a65336a78b8a9cf11f 100644 (file)
--- a/.github/workflows/ci-freebsd.yml
+++ b/.github/workflows/ci-freebsd.yml
@@ -5,6 +5,7 @@ on:
     branches-ignore:
       - coverity_scan
       - run-fuzzer**
+      - debug-fuzzer-**
   pull_request:
 
 env:

diff --git a/.github/workflows/ci-rpm.yml b/.github/workflows/ci-rpm.yml
index 806409ff1fe9993956edc067724fe1e4839bd759..1b8f48897f3ac085a40aa7740a36ad6f74a7c373 100644 (file)
--- a/.github/workflows/ci-rpm.yml
+++ b/.github/workflows/ci-rpm.yml
@@ -5,6 +5,7 @@ on:
     branches-ignore:
       - coverity_scan
       - run-fuzzer**
+      - debug-fuzzer-**
   schedule:
     - cron: '0 20 * * *'
 

diff --git a/.github/workflows/ci-scheduled-fuzzing.yml b/.github/workflows/ci-scheduled-fuzzing.yml
index 80bf1fbf00581f28a958066382194831f66bcd1d..78a0cd29df189fae73ef80150c5114d42a63f2be 100644 (file)
--- a/.github/workflows/ci-scheduled-fuzzing.yml
+++ b/.github/workflows/ci-scheduled-fuzzing.yml
@@ -14,6 +14,24 @@
 #    - 'run-fuzzer-3600':        Start fuzzing all protocols for one hour
 #    - 'run-fuzzer-radius-7200': Start fuzzing RADIUS for two hours
 #
+#  Fuzzing failures (full log output including backtraces and reproducers) are
+#  uploaded as "artifacts" for the GitHub Actions run.
+#
+#  The following script can be used to list fuzzer failures and download
+#  reproducers for local reproduction:
+#
+#      scripts/build/fuzzer-fetch-artifacts
+#
+#  If local reproduction does not recreate the failure then you may wish to
+#  attempt the reproduction within a GitHub Actions runner. To do this push to
+#  the following branch:
+#
+#    - 'debug-fuzzer-<protocol>'
+#
+#  This will perform a fuzzer-enabled build and then instead of fuzzing will
+#  launch a tmate session that can be used to access the environment. Further
+#  details at the end of this file.
+#
 
 name: Scheduled fuzzing
 
@@ -21,6 +39,7 @@ on:
   push:
     branches:
       - 'run-fuzzer**'
+      - 'debug-fuzzer-**'
 
   schedule:
     - cron: '0 4 * * *'
@@ -89,6 +108,9 @@ jobs:
           PROTOS=( "${PROTOS%-*}" )
         elif [[ "$GITHUB_REF" = refs/heads/run-fuzzer-* ]]; then
           TOTAL_RUNTIME=${GITHUB_REF#refs/heads/run-fuzzer-}
+        elif [[ "$GITHUB_REF" = refs/heads/debug-fuzzer-* ]]; then
+          PROTOS=${GITHUB_REF#refs/heads/debug-fuzzer-}
+          PROTOS=( "${PROTOS%-*}" )
         fi
         P=$(
             for i in ${!PROTOS[@]}; do
@@ -162,6 +184,7 @@ jobs:
         key: corpus-${{ matrix.env.PROTOCOL }}-${{ steps.corpusparams.outputs.corpusct }}-${{ github.run_number }}
         restore-keys: |
           corpus-${{ matrix.env.PROTOCOL }}-${{ steps.corpusparams.outputs.corpusct }}-
+      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }}
 
     - name: Package manager performance improvements
       run: |
@@ -236,6 +259,7 @@ jobs:
       env:
         GITHUB_REF: "${{ github.ref }}"
         START_TIMESTAMP: "${{ needs.set-matrix.outputs.starttimestamp }}"
+      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }}
 
     - name: "Clang libFuzzer: Store assets on failure"
       uses: actions/upload-artifact@v2
@@ -243,7 +267,7 @@ jobs:
         name: clang-fuzzer-${{ matrix.env.PROTOCOL }}-${{ steps.pick_commit.outputs.commit_id }}
         path: build/fuzzer
         retention-days: 30
-      if: ${{ failure() }}
+      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer') && failure() }}
 
     #
     #  Merge the corpus which will be stored in the cache for the next run
@@ -251,6 +275,7 @@ jobs:
     - name: Merge the corpus
       run: |
         make test.fuzzer.$PROTOCOL.merge
+      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }}
 
     #
     #  We can push the LFS file directly, but we must use the GitHub API to
@@ -273,4 +298,43 @@ jobs:
         fi
       env:
         GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
-      if: ${{ steps.corpusparams.outputs.corpusage > 2592000 && github.repository_owner == 'FreeRADIUS' }}
+      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') && steps.corpusparams.outputs.corpusage > 2592000 && github.repository_owner == 'FreeRADIUS' }}
+
+
+    #
+    #  If we are on the 'debug-fuzzer-*' branch then we start a tmate session to
+    #  provide interactive shell access to the session so that the reproducers
+    #  can be attempted in an identical environment to which the scheduled
+    #  fuzzing occurred.
+    #
+    #  The SSH rendezvous point will be emited continuously in the job output,
+    #  which will look something like:
+    #
+    #      SSH: ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io
+    #
+    #  For example:
+    #
+    #      git push origin debug-fuzzer-radius --force
+    #
+    #  Look at the job output in: https://github.com/FreeRADIUS/freeradius-server/actions
+    #
+    #      ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io
+    #
+    #  Access requires that you have the private key corresponding to the
+    #  public key of the GitHub user that initiated the job.
+    #
+    #  Within this session you can use scripts/build/fuzzer-fetch-artifacts to
+    #  download the reproducers just as you would do locally, e.g.
+    #
+    #      export GITHUB_TOKEN=<personal-access-token>
+    #      scripts/build/fuzzer-fetch-artifacts
+    #      scripts/build/fuzzer-fetch-artifacts https://api.github.com/repos/FreeRADIUS/freeradius-server/actions/artifacts/186571481/zip
+    #      scripts/build/fuzzer build/fuzzer/radius/crash-f1536d0fa2de775038e5dab74d233487a7cde819
+    #
+    - name: "Debug: Start tmate"
+      uses: mxschmitt/action-tmate@v3
+      with:
+        limit-access-to-actor: true
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      if: ${{ startsWith(github.ref, 'refs/heads/debug-fuzzer-') && always() }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0e6f90d43f529973064fb90f15a2b6b65e52b68a..4adf98b921118f80fcb9de74a3ee51f9c7b313ad 100644 (file)
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -5,6 +5,7 @@ on:
     branches-ignore:
       - coverity_scan
       - run-fuzzer**
+      - debug-fuzzer-**
   pull_request:
 
 env:

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
index 1efbb17a9e58815777d24b879821d7f4bcfb5c50..07bd06e1c47aef32c1a797e6ecc24725bf0c70ce 100644 (file)
--- a/.github/workflows/documentation.yml
+++ b/.github/workflows/documentation.yml
@@ -5,6 +5,7 @@ on:
     branches-ignore:
       - coverity_scan
       - run-fuzzer**
+      - debug-fuzzer-**
   pull_request:
 
 jobs: