ci: mov from cifuzz to clusterfuzzlite

To better support main7 CI fuzzing

Ticket: 7253

diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile
new file mode 100644 (file)
index 0000000..93e2a73
--- /dev/null
+++ b/.clusterfuzzlite/Dockerfile
@@ -0,0 +1,20 @@
+FROM gcr.io/oss-fuzz-base/base-builder-rust
+RUN apt-get update && apt-get install -y build-essential autoconf automake libtool make pkg-config  python flex bison zlib1g-dev libpcre3-dev cmake tshark
+
+# TODO libmagic, liblzma and other optional libraries
+ADD https://github.com/PhilipHazel/pcre2/releases/download/pcre2-10.44/pcre2-10.44.tar.gz pcre2-10.44.tar.gz
+ADD https://www.tcpdump.org/release/libpcap-1.10.5.tar.gz libpcap-1.10.5.tar.gz
+ADD https://github.com/akheron/jansson/releases/download/v2.14/jansson-2.14.tar.gz jansson-2.14.tar.gz
+RUN git clone --depth=1 https://github.com/yaml/libyaml
+ADD https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz lz4-1.10.0.tar.gz
+RUN git clone --depth 1 -b develop https://github.com/madler/zlib.git
+RUN git clone --depth=1 https://github.com/catenacyber/fuzzpcap
+
+ENV RUSTUP_TOOLCHAIN nightly
+RUN cargo install --force cbindgen
+
+RUN git clone --depth 1 https://github.com/OISF/libhtp.git libhtp
+
+COPY . $SRC/suricata
+WORKDIR $SRC/suricata
+COPY ./.clusterfuzzlite/build.sh $SRC/

diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh
new file mode 100755 (executable)
index 0000000..db611be
--- /dev/null
+++ b/.clusterfuzzlite/build.sh
@@ -0,0 +1,105 @@
+#!/bin/bash -eu
+
+cd $SRC/
+# build dependencies statically
+if [ "$SANITIZER" = "memory" ]
+then
+    (
+    cd zlib
+    ./configure --static
+    make -j$(nproc) clean
+    make -j$(nproc) all
+    make -j$(nproc) install
+    )
+fi
+
+(
+tar -xvzf pcre2-10.44.tar.gz
+cd pcre2-10.44
+./configure --disable-shared
+make -j$(nproc) clean
+make -j$(nproc) all
+make -j$(nproc) install
+)
+
+tar -xvzf lz4-1.10.0.tar.gz
+cd lz4-1.10.0
+make liblz4.a
+cp lib/liblz4.a /usr/local/lib/
+cp lib/lz4*.h /usr/local/include/
+cd ..
+
+tar -xvzf jansson-2.14.tar.gz
+cd jansson-2.14
+./configure --disable-shared
+make -j$(nproc)
+make install
+cd ..
+
+tar -xvzf libpcap-1.10.5.tar.gz
+cd libpcap-1.10.5
+./configure --disable-shared
+make -j$(nproc)
+make install
+cd ..
+
+cd fuzzpcap
+mkdir build
+cd build
+cmake ..
+make install
+cd ../..
+
+cd libyaml
+./bootstrap
+./configure --disable-shared
+make -j$(nproc)
+make install
+cd ..
+
+export CARGO_BUILD_TARGET="x86_64-unknown-linux-gnu"
+# cf https://github.com/google/sanitizers/issues/1389
+export MSAN_OPTIONS=strict_memcmp=false
+
+#run configure with right options
+if [ "$SANITIZER" = "address" ]
+then
+    export RUSTFLAGS="$RUSTFLAGS -Cpasses=sancov-module -Cllvm-args=-sanitizer-coverage-level=4 -Cllvm-args=-sanitizer-coverage-trace-compares -Cllvm-args=-sanitizer-coverage-inline-8bit-counters -Cllvm-args=-sanitizer-coverage-pc-table -Clink-dead-code -Cllvm-args=-sanitizer-coverage-stack-depth -Ccodegen-units=1"
+    export RUSTFLAGS="$RUSTFLAGS -Cdebug-assertions=yes"
+fi
+
+rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu
+
+#we did not put libhtp there before so that cifuzz does not remove it
+cp -r libhtp suricata/
+# build project
+
+cd suricata
+sh autogen.sh
+
+./src/tests/fuzz/oss-fuzz-configure.sh
+make -j$(nproc)
+
+./src/suricata --list-app-layer-protos | tail -n +2 | while read i; do cp src/fuzz_applayerparserparse $OUT/fuzz_applayerparserparse""_$i; done
+
+(
+cd src
+ls fuzz_* | while read i; do
+    cp $i $OUT/$i
+    # download oss-fuzz public corpuses
+    wget "https://storage.googleapis.com/suricata-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/suricata_$i/public.zip" --output-file=$OUT/"$i"_seed_corpus.zip || true
+done
+)
+# dictionaries
+./src/suricata --list-keywords | grep "\- " | sed 's/- //' | awk '{print "\""$0"\""}' > $OUT/fuzz_siginit.dict
+
+echo \"SMB\" > $OUT/fuzz_applayerparserparse""_smb.dict
+
+echo "\"FPC0\"" > $OUT/fuzz_sigpcap_aware.dict
+echo "\"FPC0\"" > $OUT/fuzz_predefpcap_aware.dict
+
+git grep tag rust | grep '"' | cut -d '"' -f2 | sort | uniq | awk 'length($0) > 2' | awk '{print "\""$0"\""}' | grep -v '\\' > generic.dict
+cat generic.dict >> $OUT/fuzz_siginit.dict
+cat generic.dict >> $OUT/fuzz_applayerparserparse.dict
+cat generic.dict >> $OUT/fuzz_sigpcap.dict
+cat generic.dict >> $OUT/fuzz_sigpcap_aware.dict

diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml
new file mode 100644 (file)
index 0000000..2606aba
--- /dev/null
+++ b/.clusterfuzzlite/project.yaml
@@ -0,0 +1,15 @@
+homepage: "https://suricata.io"
+language: rust
+primary_contact: "vjulien@openinfosecfoundation.org"
+auto_ccs:
+- "jish@openinfosecfoundation.org"
+- "p.antoine@catenacyber.fr"
+sanitizers:
+  - address
+  - memory
+  - undefined
+fuzzing_engines:
+  - afl
+  - honggfuzz
+  - libfuzzer
+main_repo: 'https://github.com/OISF/suricata.git'

diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml
index d6019cd805999e7807d522443489497d6ac99377..4d130f38c6644498183df2b4839b69f1be5951f0 100644 (file)
--- a/.github/workflows/cifuzz.yml
+++ b/.github/workflows/cifuzz.yml
@@ -1,4 +1,5 @@
-name: CIFuzz
+# CIFuzz did not support fuzzing of branch main7 well
+name: ClusterFuzzLite
 
 on:
   pull_request:
@@ -25,21 +26,18 @@ jobs:
         sudo rm -rf /usr/share/dotnet/ /usr/share/swift /usr/local/.ghcup/ /usr/local/share/powershell /usr/local/share/chromium /usr/local/lib/android /usr/local/lib/node_modules
         df
    - name: Build Fuzzers (${{ matrix.sanitizer }})
-     uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+     id: build
+     uses: google/clusterfuzzlite/actions/build_fuzzers@v1
      with:
-       oss-fuzz-project-name: 'suricata'
-       dry-run: false
+       language: rust
+       github-token: ${{ secrets.GITHUB_TOKEN }}
        sanitizer: ${{ matrix.sanitizer }}
    - name: Run Fuzzers (${{ matrix.sanitizer }})
-     uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+     id: run
+     uses: google/clusterfuzzlite/actions/run_fuzzers@v1
      with:
-       oss-fuzz-project-name: 'suricata'
+       github-token: ${{ secrets.GITHUB_TOKEN }}
        fuzz-seconds: 600
-       dry-run: false
+       mode: 'code-change'
        sanitizer: ${{ matrix.sanitizer }}
-   - name: Upload Crash
-     uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
-     if: failure()
-     with:
-       name: ${{ matrix.sanitizer }}-artifacts
-       path: ./out/artifacts
+       output-sarif: true