]> git.ipfire.org Git - thirdparty/nftables.git/commitdiff
tests: 30s-stress: add failslab and abort phase tests
authorFlorian Westphal <fw@strlen.de>
Wed, 23 Aug 2023 04:29:13 +0000 (06:29 +0200)
committerFlorian Westphal <fw@strlen.de>
Thu, 24 Aug 2023 17:03:23 +0000 (19:03 +0200)
Pablo suggested to also cover abort phase by intentionally deleting
non-existent or adding clashing keys.

While at it:
add rules with anon sets and jumps to anonymous chains and a few
constant sets.

Pick different key sizes so there is a higher chance kernel picks
different backend storages such as bitmap or hash_fast.

add failslab support, this also covers unlikely or "impossible" cases like
failing GFP_KERNEL allocations.

randomly spawn 'nft monitor' in the background for a random duration
to cover notification path.

Try to randomly delete a set or chain from control plane.

Randomly set a table as dormant (and back to normal).

Allow to pass the test runtime as argument, so one can now do

./30s-stress 3600

to have the test run for one hour.

For such long test durations, make sure the ruleset
gets regenerated periodically.

Signed-off-by: Florian Westphal <fw@strlen.de>
tests/shell/testcases/transactions/30s-stress

index 3539a3078b131811ab365aba9c5f853913d331ef..4d3317e22b0c26506ef3d512867de2824d09c111 100755 (executable)
@@ -1,12 +1,66 @@
 #!/bin/bash
 
+runtime=30
+
+# allow stand-alone execution as well, e.g. '$0 3600'
+if [ x"$1" != "x" ] ;then
+       if [ $1 -ge 0 ]; then
+               runtime="$1"
+       else
+               echo "Invalid runtime $1"
+               exit 1
+       fi
+fi
+
+if [ x = x"$NFT" ] ; then
+       NFT=nft
+fi
+
 testns=testns-$(mktemp -u "XXXXXXXX")
 tmp=""
 
-tables="foo bar baz"
-runtime=10
+faultname="/proc/self/make-it-fail"
+tables="foo bar"
+
+failslab_defaults() {
+       test -w $faultname || return
+
+       # Disable fault injection unless process has 'make-it-fail' set
+       echo Y > /sys/kernel/debug/failslab/task-filter
+
+       # allow all slabs to fail (if process is tagged).
+       find /sys/kernel/slab/ -wholename '*/kmalloc-[0-9]*/failslab' -type f -exec sh -c 'echo 1 > {}' \;
+
+       # no limit on the number of failures
+       echo -1 > /sys/kernel/debug/failslab/times
+
+       # Set to 2 for full dmesg traces for each injected error
+       echo 0 > /sys/kernel/debug/failslab/verbose
+}
+
+failslab_random()
+{
+       r=$((RANDOM%2))
+
+       if [ $r -eq 0 ]; then
+               echo Y > /sys/kernel/debug/failslab/ignore-gfp-wait
+       else
+               echo N > /sys/kernel/debug/failslab/ignore-gfp-wait
+       fi
+
+       r=$((RANDOM%5))
+       echo $r > /sys/kernel/debug/failslab/probability
+       r=$((RANDOM%100))
+       echo $r > /sys/kernel/debug/failslab/interval
+
+       # allow a small initial 'success budget'.
+       # failures only appear after this many allocated bytes.
+       r=$((RANDOM%16384))
+       echo $r > /sys/kernel/debug/$FAILTYPE/space
+}
 
 netns_del() {
+       ip netns pids "$testns" | xargs kill 2>/dev/null
        ip netns del "$testns"
 }
 
@@ -21,12 +75,30 @@ cleanup() {
        netns_del
 }
 
+nft_with_fault_inject()
+{
+       file="$1"
+
+       if [ -w "$faultname" ]; then
+               failslab_random
+
+               ip netns exec "$testns" bash -c "echo 1 > $faultname ; exec $NFT -f $file"
+       fi
+
+       ip netns exec "$testns" $NFT -f "$file"
+}
+
 trap cleanup EXIT
 tmp=$(mktemp)
 
 random_verdict()
 {
        max="$1"
+
+       if [ $max -eq 0 ]; then
+               max=1
+       fi
+
        rnd=$((RANDOM%max))
 
        if [ $rnd -gt 0 ];then
@@ -43,8 +115,8 @@ random_verdict()
 
 randsleep()
 {
-       local s=$((RANDOM%3))
-       local ms=$((RANDOM%10))
+       local s=$((RANDOM%1))
+       local ms=$((RANDOM%1000))
        sleep $s.$ms
 }
 
@@ -72,8 +144,79 @@ randdeltable()
                        r=$((RANDOM%10))
 
                        if [ $r -eq 1 ] ;then
-                               ip netns exec $testns $NFT delete table $t
+                               ip netns exec $testns $NFT delete table inet $t
+                               randsleep
+                       fi
+               done
+       done
+}
+
+randdelset()
+{
+       while [ -r $tmp ]; do
+               randsleep
+               for t in $tables; do
+                       r=$((RANDOM%10))
+                       s=$((RANDOM%10))
+
+                       case $r in
+                       0)
+                               setname=set_$s
+                               ;;
+                       1)
+                               setname=sett${s}
+                               ;;
+                       2)
+                               setname=dmap_${s}
+                               ;;
+                       3)
+                               setname=dmapt${s}
+                               ;;
+                       4)
+                               setname=vmap_${s}
+                               ;;
+                       5)
+                               setname=vmapt${s}
+                               ;;
+                       *)
+                               continue
+                               ;;
+                       esac
+
+                       if [ $r -eq 1 ] ;then
+                               ip netns exec $testns $NFT delete set inet $t $setname
+                       fi
+               done
+       done
+}
+
+randdelchain()
+{
+       while [ -r $tmp ]; do
+               for t in $tables; do
+                       local c=$((RANDOM%100))
+                       randsleep
+                       chain=$(printf "chain%03u" "$c")
+
+                       local r=$((RANDOM%10))
+                       if [ $r -eq 1 ];then
+                               # chain can be invalid/unknown.
+                               ip netns exec $testns $NFT delete chain inet $t $chain
+                       fi
+               done
+       done
+}
+
+randdisable()
+{
+       while [ -r $tmp ]; do
+               for t in $tables; do
+                       randsleep
+                       local r=$((RANDOM%10))
+                       if [ $r -eq 1 ];then
+                               ip netns exec $testns $NFT add table inet $t '{flags dormant; }'
                                randsleep
+                               ip netns exec $testns $NFT add table inet $t '{ }'
                        fi
                done
        done
@@ -89,30 +232,153 @@ randdelns()
        done
 }
 
+random_element_string=""
+
+# create a random element.  Could cause any of the following:
+# 1. Invalid set/map
+# 2. Element already exists in set/map w. create
+# 3. Element is new but wants to jump to unknown chain
+# 4. Element already exsists in set/map w. add, but verdict (map data) differs
+# 5. Element is created/added/deleted from 'flags constant' set.
+random_elem()
+{
+       tr=$((RANDOM%2))
+       t=0
+
+       for table in $tables; do
+               if [ $t -ne $tr ]; then
+                       t=$((t+1))
+                       continue
+               fi
+
+               kr=$((RANDOM%2))
+               k=0
+               cnt=0
+               for key in "single" "concat"; do
+                       if [ $k -ne $kr ] ;then
+                               cnt=$((cnt+2))
+                               k=$((k+1))
+                               continue
+                       fi
+
+                       fr=$((RANDOM%2))
+                       f=0
+                       for flags in "" "interval" ; do
+                               cnt=$((cnt+1))
+                               if [ $f -ne fkr ] ;then
+                                       f=$((f+1))
+                                       continue
+                               fi
+
+                               want="${key}${flags}"
+
+                               e=$((RANDOM%256))
+                               case "$want" in
+                               "single") element="10.1.1.$e"
+                                       ;;
+                               "concat") element="10.1.2.$e . $((RANDOM%65536))"
+                                       ;;
+                               "singleinterval") element="10.1.$e.0-10.1.$e.$e"
+                                       ;;
+                               "concatinterval") element="10.1.$e.0-10.1.$e.$e . $((RANDOM%65536))"
+                                       ;;
+                               *)      echo "bogus key $want"
+                                       exit 111
+                                       ;;
+                               esac
+
+                               # This may result in invalid jump, but thats what we want.
+                               count=$(($RANDOM%100))
+
+                               r=$((RANDOM%7))
+                               case "$r" in
+                               0)
+                                       random_element_string=" inet $table set_${cnt} { $element }"
+                                       ;;
+                               1)      random_element_string="inet $table sett${cnt} { $element timeout $((RANDOM%60))s }"
+                                       ;;
+                               2)      random_element_string="inet $table dmap_${cnt} { $element : $RANDOM }"
+                                       ;;
+                               3)      random_element_string="inet $table dmapt${cnt} { $element timeout $((RANDOM%60))s : $RANDOM }"
+                                       ;;
+                               4)      random_element_string="inet $table vmap_${cnt} { $element : `random_verdict $count` }"
+                                       ;;
+                               5)      random_element_string="inet $table vmapt${cnt} { $element timeout $((RANDOM%60))s : `random_verdict $count` }"
+                                       ;;
+                               6)      random_element_string="inet $table setc${cnt} { $element }"
+                                       ;;
+                               esac
+
+                               return
+                       done
+               done
+       done
+}
+
 randload()
 {
        while [ -r $tmp ]; do
+               random_element_string=""
                r=$((RANDOM%10))
 
-               if [ $r -eq 1 ] ;then
+               what=""
+               case $r in
+               1)
                        (echo "flush ruleset"; cat "$tmp"
                         echo "insert rule inet foo INPUT meta nftrace set 1"
                         echo "insert rule inet foo OUTPUT meta nftrace set 1"
-                       ) | ip netns exec "$testns" $NFT -f /dev/stdin
+                       ) | nft_with_fault_inject "/dev/stdin"
+                       ;;
+               2)      what="add"
+                       ;;
+               3)      what="create"
+                       ;;
+               4)      what="delete"
+                       ;;
+               5)      what="destroy"
+                       ;;
+               6)      what="get"
+                       ;;
+               *)
+                       randsleep
+                       ;;
+               esac
+
+               if [ x"$what" = "x" ]; then
+                       nft_with_fault_inject "$tmp"
                else
-                       ip netns exec "$testns" $NFT -f "$tmp"
+                       # This can trigger abort path, for various reasons:
+                       # invalid set name
+                       # key mismatches set specification (concat vs. single value)
+                       # attempt to delete non-existent key
+                       # attempt to create dupliacte key
+                       # attempt to add duplicate key with non-matching value (data)
+                       # attempt to add new uniqeue key with a jump to an unknown chain
+                       random_elem
+                       ( cat "$tmp"; echo "$what element $random_element_string") | nft_with_fault_inject "/dev/stdin"
                fi
+       done
+}
 
+randmonitor()
+{
+       while [ -r $tmp ]; do
                randsleep
+               timeout=$((RANDOM%16))
+               timeout $((timeout+1)) $NFT monitor > /dev/null
        done
 }
 
 floodping() {
        cpunum=$(grep -c processor /proc/cpuinfo)
+       cpunum=$((cpunum+1))
 
        while [ -r $tmp ]; do
+               spawn=$((RANDOM%$cpunum))
+
+               # spawn at most $cpunum processes. Or maybe none at all.
                i=0
-               while [ $i -lt $cpunum ]; do
+               while [ $i -lt $spawn ]; do
                        mask=$(printf 0x%x $((1<<$i)))
                        timeout 3 ip netns exec "$testns" taskset $mask ping -4 -fq 127.0.0.1 > /dev/null &
                        timeout 3 ip netns exec "$testns" taskset $mask ping -6 -fq ::1 > /dev/null &
@@ -126,15 +392,33 @@ floodping() {
 
 stress_all()
 {
+       # if fault injection is enabled, first a quick test to trigger
+       # abort paths without any parallel deletes/flushes.
+       if [ -w $faultname ] ;then
+               for i in $(seq 1 10);do
+                       nft_with_fault_inject "$tmp"
+               done
+       fi
+
        randlist &
        randflush &
        randdeltable &
+       randdisable &
+       randdelchain &
+       randdelset &
        randdelns &
        randload &
+       randmonitor &
 }
 
+gen_ruleset() {
+echo > "$tmp"
 for table in $tables; do
-       count=$((RANDOM % 200))
+       count=$((RANDOM % 100))
+       if [ $count -lt 1 ];then
+               count=1
+       fi
+
        echo add table inet "$table" >> "$tmp"
        echo flush table inet "$table" >> "$tmp"
 
@@ -145,15 +429,61 @@ for table in $tables; do
                echo "add chain inet $table $chain" >> "$tmp"
        done
 
+       echo "add chain inet $table defaultchain" >> "$tmp"
+
        for c in $(seq 1 $count); do
                chain=$(printf "chain%03u" "$c")
                for BASE in INPUT OUTPUT; do
                        echo "add rule inet $table $BASE counter jump $chain" >> "$tmp"
                done
-               echo "add rule inet $table $chain counter return" >> "$tmp"
+               if [ $((RANDOM%10)) -eq 1 ];then
+                       echo "add rule inet $table $chain counter jump defaultchain" >> "$tmp"
+               else
+                       echo "add rule inet $table $chain counter return" >> "$tmp"
+               fi
        done
 
        cnt=0
+
+       # add a few anonymous sets. rhashtable is convered by named sets below.
+       c=$((RANDOM%$count))
+       chain=$(printf "chain%03u" "$((c+1))")
+       echo "insert rule inet $table $chain tcp dport 22-26 ip saddr { 1.2.3.4, 5.6.7.8 } counter comment hash_fast" >> "$tmp"
+       echo "insert rule inet $table $chain ip6 saddr { ::1, dead::beef } counter" comment hash >> "$tmp"
+       echo "insert rule inet $table $chain ip saddr { 1.2.3.4 - 5.6.7.8, 127.0.0.1 } comment rbtree" >> "$tmp"
+       # bitmap 1byte, with anon chain jump
+       echo "insert rule inet $table $chain ip protocol { 6, 17 } jump { jump defaultchain; counter; }" >> "$tmp"
+       # bitmap 2byte
+       echo "insert rule inet $table $chain tcp dport != { 22, 23, 80 } goto defaultchain" >> "$tmp"
+       echo "insert rule inet $table $chain tcp dport { 1-1024, 8000-8080 } jump defaultchain comment rbtree" >> "$tmp"
+       # pipapo (concat + set), with goto anonymous chain.
+       echo "insert rule inet $table $chain ip saddr . tcp dport { 1.2.3.4 . 1-1024, 1.2.3.6 - 1.2.3.10 . 8000-8080, 1.2.3.4 . 8080, 1.2.3.6 - 1.2.3.10 . 22 } goto { jump defaultchain; counter; }" >> "$tmp"
+
+       # add a few anonymous sets. rhashtable is convered by named sets below.
+       c=$((RANDOM%$count))
+       chain=$(printf "chain%03u" "$((c+1))")
+       echo "insert rule inet $table $chain tcp dport 22-26 ip saddr { 1.2.3.4, 5.6.7.8 } counter comment hash_fast" >> "$tmp"
+       echo "insert rule inet $table $chain ip6 saddr { ::1, dead::beef } counter" comment hash >> "$tmp"
+       echo "insert rule inet $table $chain ip saddr { 1.2.3.4 - 5.6.7.8, 127.0.0.1 } comment rbtree" >> "$tmp"
+       # bitmap 1byte, with anon chain jump
+       echo "insert rule inet $table $chain ip protocol { 6, 17 } jump { jump defaultchain; counter; }" >> "$tmp"
+       # bitmap 2byte
+       echo "insert rule inet $table $chain tcp dport != { 22, 23, 80 } goto defaultchain" >> "$tmp"
+       echo "insert rule inet $table $chain tcp dport { 1-1024, 8000-8080 } jump defaultchain comment rbtree" >> "$tmp"
+       # pipapo (concat + set), with goto anonymous chain.
+       echo "insert rule inet $table $chain ip saddr . tcp dport { 1.2.3.4 . 1-1024, 1.2.3.6 - 1.2.3.10 . 8000-8080, 1.2.3.4 . 8080, 1.2.3.6 - 1.2.3.10 . 22 } goto { jump defaultchain; counter; }" >> "$tmp"
+
+       # add constant/immutable sets
+       size=$((RANDOM%5120000))
+       size=$((size+2))
+       echo "add set inet $table setc1 { typeof tcp dport; size $size; flags constant; elements = { 22, 44 } }" >> "$tmp"
+       echo "add set inet $table setc2 { typeof ip saddr; size $size; flags constant; elements = { 1.2.3.4, 5.6.7.8 } }" >> "$tmp"
+       echo "add set inet $table setc3 { typeof ip6 daddr; size $size; flags constant; elements = { ::1, dead::1 } }" >> "$tmp"
+       echo "add set inet $table setc4 { typeof tcp dport; size $size; flags interval,constant; elements = { 22-44, 55-66 } }" >> "$tmp"
+       echo "add set inet $table setc5 { typeof ip saddr; size $size; flags interval,constant; elements = { 1.2.3.4-5.6.7.8, 10.1.1.1 } }" >> "$tmp"
+       echo "add set inet $table setc6 { typeof ip6 daddr; size $size; flags interval,constant; elements = { ::1, dead::1-dead::3 } }" >> "$tmp"
+
+       # add named sets with various combinations (plain value, range, concatenated values, concatenated ranges, with timeouts, with data ...)
        for key in "ip saddr" "ip saddr . tcp dport"; do
                for flags in "" "flags interval;" ; do
                        timeout=$((RANDOM%10))
@@ -175,11 +505,15 @@ for table in $tables; do
                for flags in "" "interval" ; do
                        want="${key}${flags}"
                        cnt=$((cnt+1))
+                       maxip=$((RANDOM%256))
 
-                       for e in $(seq 1 255);do
+                       if [ $maxip -eq 0 ];then
+                               maxip=1
+                       fi
+
+                       for e in $(seq 1 $maxip);do
                                case "$want" in
-                               "single")
-                                       element="10.1.1.$e"
+                               "single") element="10.1.1.$e"
                                        ;;
                                "concat")
                                        element="10.1.2.$e . $((RANDOM%65536))"
@@ -206,18 +540,44 @@ for table in $tables; do
                done
        done
 done
+}
+
+run_test()
+{
+       local time_now=$(date +%s)
+       local time_stop=$((time_now + $runtime))
+       local regen=30
+
+       while [ $time_now -lt $time_stop ]; do
+               if [ $regen -gt 0 ];then
+                       sleep 1
+                       time_now=$(date +%s)
+                       regen=$((regen-1))
+                       continue
+               fi
+
+               # This clobbers the previously generated ruleset, this is intentional.
+               gen_ruleset
+               regen=$((RANDOM%60))
+               regen=$((regen+2))
+               time_now=$(date +%s)
+       done
+}
 
 netns_add
 
+gen_ruleset
 ip netns exec "$testns" $NFT -f "$tmp" || exit 1
 
+failslab_defaults
+
 stress_all 2>/dev/null &
 
 randsleep
 
 floodping 2> /dev/null &
 
-sleep $runtime
+run_test
 
 # this stops stress_all
 rm -f "$tmp"