From: Victor Julien Date: Tue, 23 Apr 2024 15:19:32 +0000 (+0200) Subject: github-ci: add af-packet and dpdk codecov builds X-Git-Tag: suricata-8.0.0-beta1~1350 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ed9ad0048d0c6b72448383714b4a867187ef1131;p=thirdparty%2Fsuricata.git github-ci: add af-packet and dpdk codecov builds Adds live tests for DPDK and AF_PACKET, with support for code coverage. --- diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index d17b4fa7e3..1fe713eac8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -1544,6 +1544,152 @@ jobs: fail_ci_if_error: false flags: unittests + ubuntu-22-04-cov-afpdpdk: + name: Ubuntu 22.04 (afpacket and dpdk coverage) + runs-on: ubuntu-latest + container: + image: ubuntu:22.04 + options: --privileged + needs: [prepare-deps, prepare-cbindgen] + steps: + - name: Cache ~/.cargo + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 + with: + path: ~/.cargo + key: ${{ github.job }}-cargo + - name: Determine number of CPUs + run: echo CPUS=$(nproc --all) >> $GITHUB_ENV + + - name: Install dependencies + run: | + apt update + apt -y install \ + libpcre2-dev \ + build-essential \ + autoconf \ + automake \ + llvm-14-dev \ + clang-14 \ + git \ + jq \ + inetutils-ping \ + libc++-dev \ + libc++abi-dev \ + libtool \ + libpcap-dev \ + libnet1-dev \ + libyaml-0-2 \ + libyaml-dev \ + libcap-ng-dev \ + libcap-ng0 \ + libmagic-dev \ + libnetfilter-queue-dev \ + libnetfilter-queue1 \ + libnfnetlink-dev \ + libnfnetlink0 \ + libnuma-dev \ + libhiredis-dev \ + liblua5.1-dev \ + libjansson-dev \ + libevent-dev \ + libevent-pthreads-2.1-7 \ + libpython2.7 \ + make \ + parallel \ + python3-yaml \ + software-properties-common \ + sudo \ + zlib1g \ + zlib1g-dev \ + exuberant-ctags \ + unzip \ + curl \ + time \ + wget \ + dpdk-dev + # specific version to match up to the llvm version in ubuntu below + - name: Install Rust + run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.63.0 -y + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - run: git config --global --add safe.directory /__w/suricata/suricata + - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + with: + name: prep + path: prep + - run: tar xf prep/libhtp.tar.gz + - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + with: + name: cbindgen + path: prep + - name: Setup cbindgen + run: | + mkdir -p $HOME/.cargo/bin + cp prep/cbindgen $HOME/.cargo/bin + chmod 755 $HOME/.cargo/bin/cbindgen + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + - name: Fix kernel mmap rnd bits + # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with + # high-entropy ASLR in much newer kernels that GitHub runners are + # using leading to random crashes: https://github.com/actions/runner-images/issues/9491 + run: sudo sysctl vm.mmap_rnd_bits=28 + - run: ./autogen.sh + - run: ./configure --with-gnu-ld --enable-dpdk --disable-shared --enable-gccprotect --localstatedir=/var --prefix=/usr --sysconfdir=/etc + env: + CC: "clang-14" + CXX: "clang++-14" + RUSTFLAGS: "-C instrument-coverage" + CFLAGS: "-fprofile-instr-generate -fcoverage-mapping -O0 -g -fno-strict-aliasing" + CXXFLAGS: "-fprofile-instr-generate -fcoverage-mapping -O0 -g -fno-strict-aliasing" + ac_cv_func_malloc_0_nonnull: "yes" + ac_cv_func_realloc_0_nonnull: "yes" + - run: make -j ${{ env.CPUS }} + env: + CC: "clang-14" + CXX: "clang++-14" + RUSTFLAGS: "-C instrument-coverage" + # IDS config + - run: rm -f ./eve.json + - run: | + timeout --kill-after=30 --preserve-status 10 \ + ./src/suricata -c .github/workflows/dpdk/suricata-null-ids.yaml -S /dev/null -l ./ --dpdk -vvvv + env: + LLVM_PROFILE_FILE: "/tmp/dpdk-ids.profraw" + - run: | + test $(jq -c 'select(.event_type == "stats")' ./eve.json | tail -n1 | jq '.stats.capture.packets > 0') = true + # IPS config + - run: rm -f ./eve.json + - run: | + timeout --kill-after=30 --preserve-status 10 \ + ./src/suricata -c .github/workflows/dpdk/suricata-null-ips.yaml -S /dev/null -l ./ --dpdk -vvvv + env: + LLVM_PROFILE_FILE: "/tmp/dpdk-ips.profraw" + - run: | + test $(jq -c 'select(.event_type == "stats")' ./eve.json | tail -n1 | jq '.stats.capture.packets > 0') = true + # AF_PACKET tests + - run: | + ./.github/workflows/live/afp-ids.sh "2" "autofp" + env: + LLVM_PROFILE_FILE: "/tmp/afp2-ids-autofp.profraw" + - run: | + ./.github/workflows/live/afp-ids.sh "2" "workers" + env: + LLVM_PROFILE_FILE: "/tmp/afp2-ids-workers.profraw" + - run: | + ./.github/workflows/live/afp-ids.sh "3" "autofp" + env: + LLVM_PROFILE_FILE: "/tmp/afp3-ids-autofp.profraw" + - run: | + ./.github/workflows/live/afp-ids.sh "3" "workers" + env: + LLVM_PROFILE_FILE: "/tmp/afp3-ids-workers.profraw" + - run: llvm-profdata-14 merge -o default.profdata $(find /tmp/ -name '*.profraw') + - run: llvm-cov-14 show ./src/suricata -instr-profile=default.profdata --show-instantiations --ignore-filename-regex="^/root/.*" > coverage.txt + - name: Upload coverage to Codecov + uses: codecov/codecov-action@c16abc29c95fcf9174b58eb7e1abf4c866893bc8 + with: + fail_ci_if_error: false + flags: livemode + ubuntu-22-04-cov-fuzz: name: Ubuntu 22.04 (fuzz corpus coverage) runs-on: ubuntu-latest diff --git a/.github/workflows/live/afp-ids.sh b/.github/workflows/live/afp-ids.sh new file mode 100755 index 0000000000..7df2cc81a7 --- /dev/null +++ b/.github/workflows/live/afp-ids.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# Script to test live IDS capabilities for AF_PACKET. Starts a ping, starts suricata, +# checks stats and alerts. Then issues a reload with a new rule file, checks stats and +# new alerts. Then shuts suricata down. + +# Call with following arguments: +# 1st: "2" or "3" to indicate the tpacket version. +# 2nd: runmode string (single/autofp/workers) + +#set -e +set -x + +if [ $# -ne "2" ]; then + echo "ERROR call with 2 args: tpacket version (2/3) and runmode (single/autofp/workers)" + exit 1; +fi + +TPACKET=$1 +RUNMODE=$2 + +# dump some info +uname -a +ip r + +# remove eve.json from previous run +if [ -f eve.json ]; then + rm eve.json +fi + +RES=0 + +# Get listen interface and "ping" target address +IFACE=$(ip r|grep default|awk '{print $5}') +echo $IFACE +GW=$(ip r|grep default|awk '{print $3}') +echo $GW + +ping $GW & +PINGPID=$! + +# set first rule file +cp .github/workflows/live/icmp.rules suricata.rules + +if [ $TPACKET = "2" ]; then + V3=true +else + V3=false +fi + +# Start Suricata, SIGINT after 120 secords. Will close it earlier through +# the unix socket. +timeout --kill-after=240 --preserve-status 120 \ + ./src/suricata -c suricata.yaml -l ./ --af-packet=$IFACE -v --set af-packet.1.tpacket-v3=$V3 --set default-rule-path=. --runmode=$RUNMODE & +SURIPID=$! + +sleep 15 + +# check stats and alerts +STATSCHECK=$(jq -c 'select(.event_type == "stats")' ./eve.json | tail -n1 | jq '.stats.capture.kernel_packets > 0') +if [ $STATSCHECK = false ]; then + echo "ERROR no packets captured" + RES=1 +fi +SID1CHECK=$(jq -c 'select(.event_type == "alert")' ./eve.json | tail -n1 | jq '.alert.signature_id == 1') +if [ $SID1CHECK = false ]; then + echo "ERROR no alerts for sid 1" + RES=1 +fi + +echo "SURIPID $SURIPID PINGPID $PINGPID" + +# set second rule file for the reload +cp .github/workflows/live/icmp2.rules suricata.rules + +# trigger the reload +export PYTHONPATH=python/ +python3 python/bin/suricatasc -c "reload-rules" /var/run/suricata/suricata-command.socket + +sleep 15 + +# check stats and alerts +STATSCHECK=$(jq -c 'select(.event_type == "stats")' ./eve.json | tail -n1 | jq '.stats.capture.kernel_packets > 0') +if [ $STATSCHECK = false ]; then + echo "ERROR no packets captured" + RES=1 +fi +SID2CHECK=$(jq -c 'select(.event_type == "alert")' ./eve.json | tail -n1 | jq '.alert.signature_id == 2') +if [ $SID2CHECK = false ]; then + echo "ERROR no alerts for sid 2" + RES=1 +fi + +kill -INT $PINGPID +wait $PINGPID +python3 python/bin/suricatasc -c "shutdown" /var/run/suricata/suricata-command.socket +wait $SURIPID + +echo "done: $RES" +exit $RES diff --git a/.github/workflows/live/icmp.rules b/.github/workflows/live/icmp.rules new file mode 100644 index 0000000000..c0f94ab545 --- /dev/null +++ b/.github/workflows/live/icmp.rules @@ -0,0 +1 @@ +alert icmp any any -> any any (itype:8; sid:1;) diff --git a/.github/workflows/live/icmp2.rules b/.github/workflows/live/icmp2.rules new file mode 100644 index 0000000000..f1a0031857 --- /dev/null +++ b/.github/workflows/live/icmp2.rules @@ -0,0 +1 @@ +alert icmp any any -> any any (itype:8; sid:2;)