From: Pablo Neira Ayuso Date: Sun, 27 Jul 2025 21:27:14 +0000 (+0200) Subject: Backport nftables tests/shell from 2a38f458f12b X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=249c6979819a5f061b24e53c2f2bd5dab679da0d;p=thirdparty%2Fnftables.git Backport nftables tests/shell from 2a38f458f12b Originally, the plan was to use tests/shell from git HEAD, but this does not work well in practise because slight changes in the test output break checks resulting in lots of [DUMP FAIL] errors. It is easier to test infrastructure self-contained in this 1.0.6.y branch. However, backporting the tests/shell into 1.0.6.y turns out to be more complicated than expected, so I decided to follow the opposite, which is to (brute) force a copy of tests/shell from 2a38f458f12bc032dac1b3ba63f95ca5a3c03fbd into this branch. This also requires a number follow up partial reverts on tests/shell updates to work with 1.0.6.y. Signed-off-by: Pablo Neira Ayuso --- diff --git a/tests/shell/features/bitshift.nft b/tests/shell/features/bitshift.nft new file mode 100644 index 000000000..7f9ccb64f --- /dev/null +++ b/tests/shell/features/bitshift.nft @@ -0,0 +1,7 @@ +# 567d746b55bc ("netfilter: bitwise: add support for shifts.") +# v5.6-rc1~151^2~73^2 +table ip t { + chain c { + meta mark set meta mark << 2 + } +} diff --git a/tests/shell/features/bitwise_multireg.nft b/tests/shell/features/bitwise_multireg.nft new file mode 100644 index 000000000..cfce5a39b --- /dev/null +++ b/tests/shell/features/bitwise_multireg.nft @@ -0,0 +1,5 @@ +table inet test { + chain y { + ct mark set ct mark | meta mark + } +} diff --git a/tests/shell/features/catchall_element.nft b/tests/shell/features/catchall_element.nft new file mode 100644 index 000000000..1a02fd614 --- /dev/null +++ b/tests/shell/features/catchall_element.nft @@ -0,0 +1,8 @@ +# aaa31047a6d2 ("netfilter: nftables: add catch-all set element support") +# v5.13-rc1~94^2~10^2~2 +table t { + map m { + type inet_service : inet_service + elements = { * : 42 } + } +} diff --git a/tests/shell/features/cgroupsv2.nft b/tests/shell/features/cgroupsv2.nft new file mode 100644 index 000000000..b6a3869db --- /dev/null +++ b/tests/shell/features/cgroupsv2.nft @@ -0,0 +1,7 @@ +# e0bb96db96f8 ("netfilter: nft_socket: add support for cgroupsv2") +# v5.13-rc1~94^2~35^2~20 +table ip t { + chain c { + socket cgroupv2 level 1 "user.slice" counter + } +} diff --git a/tests/shell/features/chain_binding.nft b/tests/shell/features/chain_binding.nft new file mode 100644 index 000000000..b381ec540 --- /dev/null +++ b/tests/shell/features/chain_binding.nft @@ -0,0 +1,7 @@ +# d0e2c7de92c7 ("netfilter: nf_tables: add NFT_CHAIN_BINDING") +# v5.9-rc1~133^2~302^2~1 +table ip t { + chain c { + jump { counter; } + } +} diff --git a/tests/shell/features/comment.sh b/tests/shell/features/comment.sh new file mode 100755 index 000000000..0ad24d04c --- /dev/null +++ b/tests/shell/features/comment.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# 002f21765320 ("netfilter: nf_tables: add userdata attributes to nft_chain") +# v5.10-rc1~107^2~60^2~5 + +EXPECTED="table ip x { + chain y { + comment \"test\" + } +}" + +$NFT -f - <<< $EXPECTED + +diff -u <($NFT list ruleset) - <<<"$EXPECTED" diff --git a/tests/shell/features/ctexpect.nft b/tests/shell/features/ctexpect.nft new file mode 100644 index 000000000..02c3dfd74 --- /dev/null +++ b/tests/shell/features/ctexpect.nft @@ -0,0 +1,10 @@ +# 857b46027d6f ("netfilter: nft_ct: add ct expectations support") +# v5.3-rc1~140^2~153^2~19 +table t { + ct expectation ctexpect { + protocol tcp + dport 5432 + timeout 1h + size 12; + } +} diff --git a/tests/shell/features/cttimeout.nft b/tests/shell/features/cttimeout.nft new file mode 100644 index 000000000..4be58cd3c --- /dev/null +++ b/tests/shell/features/cttimeout.nft @@ -0,0 +1,8 @@ +# 7e0b2b57f01d ("netfilter: nft_ct: add ct timeout support") +# v4.19-rc1~140^2~64^2~3 +table t { + ct timeout cttime { + protocol tcp; + policy = {established: 120 } + } +} diff --git a/tests/shell/features/curl.sh b/tests/shell/features/curl.sh new file mode 100755 index 000000000..6aa51623a --- /dev/null +++ b/tests/shell/features/curl.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# check whether curl is installed and supports ftp +curl --version | grep "^Protocols: "| grep -q " ftp" diff --git a/tests/shell/features/destroy.nft b/tests/shell/features/destroy.nft new file mode 100644 index 000000000..b97242e41 --- /dev/null +++ b/tests/shell/features/destroy.nft @@ -0,0 +1,3 @@ +# f80a612dd77c ("netfilter: nf_tables: add support to destroy operation") +# v6.3-rc1~162^2~264^2 +destroy table t diff --git a/tests/shell/features/dynset_op_delete.nft b/tests/shell/features/dynset_op_delete.nft new file mode 100644 index 000000000..125b4526b --- /dev/null +++ b/tests/shell/features/dynset_op_delete.nft @@ -0,0 +1,12 @@ +# d0a8d877da97 ("netfilter: nft_dynset: support for element deletion") +# v5.4-rc1~131^2~59^2~4 +table ip x { + set s { + flags dynamic; + type inet_service; + } + + chain y { + delete @s { tcp dport } + } +} diff --git a/tests/shell/features/elem_timeout_update.sh b/tests/shell/features/elem_timeout_update.sh new file mode 100755 index 000000000..6243170a6 --- /dev/null +++ b/tests/shell/features/elem_timeout_update.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# 4201f3938914 ("netfilter: nf_tables: set element timeout update support") + +$NFT -f - </dev/null 2>&1 diff --git a/tests/shell/features/stateful_object_update.sh b/tests/shell/features/stateful_object_update.sh new file mode 100755 index 000000000..62fbf7e38 --- /dev/null +++ b/tests/shell/features/stateful_object_update.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# d62d0ba97b58 ("netfilter: nf_tables: Introduce stateful object update operation") +# v5.4-rc1~131^2~59^2~2 + +set -e +$NFT add table test-ip +$NFT add quota test-ip traffic-quota 25 mbytes +$NFT add quota test-ip traffic-quota 50 mbytes + +EXPECTED="table ip test-ip { + quota traffic-quota { + 50 mbytes + } +}" + +GET="$($NFT list ruleset)" +if [ "$EXPECTED" != "$GET" ] ; then + diff -u <(echo "$EXPECTED") <(echo "$GET") + exit 1 +fi diff --git a/tests/shell/features/synproxy.nft b/tests/shell/features/synproxy.nft new file mode 100644 index 000000000..bea4f9205 --- /dev/null +++ b/tests/shell/features/synproxy.nft @@ -0,0 +1,9 @@ +# v5.3-rc1~140^2~44^2~10 +# ad49d86e07a4 ("netfilter: nf_tables: Add synproxy support") +table inet x { + synproxy https-synproxy { + mss 1460 + wscale 7 + timestamp sack-perm + } +} diff --git a/tests/shell/features/table_flag_owner.nft b/tests/shell/features/table_flag_owner.nft new file mode 100644 index 000000000..aef122a07 --- /dev/null +++ b/tests/shell/features/table_flag_owner.nft @@ -0,0 +1,5 @@ +# 6001a930ce03 ("netfilter: nftables: introduce table ownership") +# v5.12-rc1~200^2~6^2 +table t { + flags owner; +} diff --git a/tests/shell/features/table_flag_persist.nft b/tests/shell/features/table_flag_persist.nft new file mode 100644 index 000000000..0da3e6d4f --- /dev/null +++ b/tests/shell/features/table_flag_persist.nft @@ -0,0 +1,3 @@ +table t { + flags persist; +} diff --git a/tests/shell/features/tcpdump.sh b/tests/shell/features/tcpdump.sh new file mode 100755 index 000000000..70df9f681 --- /dev/null +++ b/tests/shell/features/tcpdump.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# check whether tcpdump is installed +tcpdump -h >/dev/null 2>&1 diff --git a/tests/shell/features/vsftpd.sh b/tests/shell/features/vsftpd.sh new file mode 100755 index 000000000..d35006405 --- /dev/null +++ b/tests/shell/features/vsftpd.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# check whether vsftpd is installed +which vsftpd >/dev/null 2>&1 diff --git a/tests/shell/helpers/json-diff-pretty.sh b/tests/shell/helpers/json-diff-pretty.sh new file mode 100755 index 000000000..bebb7e8ed --- /dev/null +++ b/tests/shell/helpers/json-diff-pretty.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +BASEDIR="$(dirname "$0")" + +[ $# -eq 2 ] || (echo "$0: expects two JSON files as arguments" ; exit 1) + +FILE1="$1" +FILE2="$2" + +pretty() +{ + "$BASEDIR/json-pretty.sh" < "$1" 2>&1 || : +} + +echo "Cmd: \"$0\" \"$FILE1\" \"$FILE2\"" +diff -u "$FILE1" "$FILE2" 2>&1 || : +diff -u <(pretty "$FILE1") <(pretty "$FILE2") 2>&1 || : diff --git a/tests/shell/helpers/json-pretty.sh b/tests/shell/helpers/json-pretty.sh new file mode 100755 index 000000000..5407a8420 --- /dev/null +++ b/tests/shell/helpers/json-pretty.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e + +exec_pretty() { + # The output of this command must be stable (and `jq` and python + # fallback must generate the same output. + + if command -v jq &>/dev/null ; then + # If we have, use `jq` + exec jq + fi + + # Fallback to python. + exec python -c ' +import json +import sys + +parsed = json.load(sys.stdin) +print(json.dumps(parsed, indent=2)) +' +} + +[ "$#" -le 1 ] || { echo "At most one argument supported" ; exit 1 ; } + +if [ "$#" -eq 1 ] ; then + # One argument passed. This must be a JSON file. + [ -f "$1" ] || { echo "File \"$1\" does not exist" ; exit 1 ; } + exec_pretty < "$1" +fi + +exec_pretty diff --git a/tests/shell/helpers/json-sanitize-ruleset.sh b/tests/shell/helpers/json-sanitize-ruleset.sh new file mode 100755 index 000000000..31b85cbd8 --- /dev/null +++ b/tests/shell/helpers/json-sanitize-ruleset.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e + +die() { + printf "%s\n" "$*" + exit 1 +} + +do_sed() { + # Normalize the "version"/"release_name", otherwise we have to + # regenerate the JSON output upon new release. + # + # Also, "handle" are not stable. Normalize them 0. + sed \ + -e '1s/^\({"nftables": \[{"metainfo": {"version": "\)[0-9.]\+\(", "release_name": "\)[^"]\+\(", "\)/\1VERSION\2RELEASE_NAME\3/' \ + -e '1s/"handle": [0-9]\+\>/"handle": 0/g' \ + "$@" +} + +if [ "$#" = 0 ] ; then + do_sed + exit $? +fi + +for f ; do + test -f "$f" || die "$0: file \"$f\" does not exist" +done + +for f ; do + do_sed -i "$f" || die "$0: \`sed -i\` failed for \"$f\"" +done diff --git a/tests/shell/helpers/nft-valgrind-wrapper.sh b/tests/shell/helpers/nft-valgrind-wrapper.sh new file mode 100755 index 000000000..98bbdf435 --- /dev/null +++ b/tests/shell/helpers/nft-valgrind-wrapper.sh @@ -0,0 +1,31 @@ +#!/bin/bash -e + +SUFFIX="$(date "+%H%M%S.%6N").$$" + +rc=0 +libtool \ + --mode=execute \ + valgrind \ + --log-file="$NFT_TEST_TESTTMPDIR/valgrind.$SUFFIX.%p.log" \ + --trace-children=yes \ + --leak-check=full \ + --show-leak-kinds=all \ + --num-callers=100 \ + --error-exitcode=122 \ + --vgdb-prefix="$_NFT_TEST_VALGRIND_VGDB_PREFIX-$SUFFIX" \ + $NFT_TEST_VALGRIND_OPTS \ + "$NFT_REAL" \ + "$@" \ + || rc=$? + +if [ "$rc" -eq 122 ] ; then + shopt -s nullglob + FILES=( "$NFT_TEST_TESTTMPDIR/valgrind.$SUFFIX."*".log" ) + shopt -u nullglob + ( + printf '%s\n' "args: $*" + printf '%s\n' "${FILES[*]}" + ) >> "$NFT_TEST_TESTTMPDIR/rc-failed-valgrind" +fi + +exit $rc diff --git a/tests/shell/helpers/random-source.sh b/tests/shell/helpers/random-source.sh new file mode 100755 index 000000000..91a8248be --- /dev/null +++ b/tests/shell/helpers/random-source.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Commands like `sort` and `shuf` have a "--random-source" argument, for +# generating a stable, reproducible output. However, they require an input +# that provides sufficiently many bytes (depending on the input). +# +# This script generates a stream that can be used like +# +# shuf --random-source=<($0 "$seed") + +seed="" +for a; do + seed="$seed${#a}:$a\n" +done + +if command -v openssl &>/dev/null ; then + # We have openssl. Use it. + # https://www.gnu.org/software/coreutils/manual/html_node/Random-sources.html#Random-sources + # + # Note that we don't care that different installations/architectures generate the + # same output. + openssl enc -aes-256-ctr -pass "pass:$seed" -nosalt /dev/null +else + # Hack something. It's much slower. + idx=0 + while : ; do + idx="$((idx++))" + seed="$(sha256sum <<<"$idx.$seed")" + echo ">>>$seed" >> a + seed="${seed%% *}" + LANG=C awk -v s="$seed" 'BEGIN{ + for (i=1; i <= length(s); i+=2) { + xchar = substr(s, i, 2); + decnum = strtonum("0x"xchar); + printf("%c", decnum); + } + }' || break + done +fi +exit 0 diff --git a/tests/shell/helpers/test-wrapper.sh b/tests/shell/helpers/test-wrapper.sh new file mode 100755 index 000000000..4a7e8b7b9 --- /dev/null +++ b/tests/shell/helpers/test-wrapper.sh @@ -0,0 +1,328 @@ +#!/bin/bash -e + +# This wrapper wraps the invocation of the test. It is called by run-tests.sh, +# and already in the unshared namespace. +# +# For some printf debugging, you can also patch this file. + +array_contains() { + local needle="$1" + local a + shift + for a; do + [ "$a" = "$needle" ] && return 0 + done + return 1 +} + +show_file() { + local filename="$1" + shift + local msg="$*" + + printf '%s\n>>>>\n' "$msg" + cat "$filename" + printf "<<<<\n" +} + +json_pretty() { + "$NFT_TEST_BASEDIR/helpers/json-pretty.sh" "$@" 2>&1 || : +} + +TEST="$1" +TESTBASE="$(basename "$TEST")" +TESTDIR="$(dirname "$TEST")" + +START_TIME="$(cut -d ' ' -f1 /proc/uptime)" + +export TMPDIR="$NFT_TEST_TESTTMPDIR" + +CLEANUP_UMOUNT_VAR_RUN=n + +cleanup() { + if [ "$CLEANUP_UMOUNT_VAR_RUN" = y ] ; then + umount "/var/run" &>/dev/null || : + fi +} + +trap cleanup EXIT + +printf '%s\n' "$TEST" > "$NFT_TEST_TESTTMPDIR/name" + +read tainted_before < /proc/sys/kernel/tainted + +if [ "$NFT_TEST_HAS_UNSHARED_MOUNT" = y ] ; then + # We have a private mount namespace. We will mount /var/run/ as a tmpfs. + # + # The main purpose is so that we can create /var/run/netns, which is + # required for `ip netns add` to work. When running as rootless, this + # is necessary to get such tests to pass. When running rootful, it's + # still useful to not touch the "real" /var/run/netns of the system. + # + # Note that this also hides everything that might reside in /var/run. + # That is desirable, as tests should not depend on content there (or if + # they do, we need to explicitly handle it as appropriate). + if mount -t tmpfs --make-private tmpfs "/var/run" 2>/dev/null ; then + CLEANUP_UMOUNT_VAR_RUN=y + fi + mkdir -p /var/run/netns +fi + +TEST_TAGS_PARSED=0 +ensure_TEST_TAGS() { + if [ "$TEST_TAGS_PARSED" = 0 ] ; then + TEST_TAGS_PARSED=1 + TEST_TAGS=( $(sed -n '1,10 { s/^.*\<\(NFT_TEST_REQUIRES\|NFT_TEST_SKIP\)\>\s*(\s*\(NFT_TEST_SKIP_[a-zA-Z0-9_]\+\|NFT_TEST_HAVE_[a-zA-Z0-9_]\+\)\s*).*$/\1(\2)/p }' "$1" 2>/dev/null || : ) ) + fi +} + +rc_test=0 + +if [ "$rc_test" -eq 0 ] ; then + for KEY in $(compgen -v | grep '^NFT_TEST_HAVE_') ; do + if [ "${!KEY}" != n ]; then + continue + fi + ensure_TEST_TAGS "$TEST" + if array_contains "NFT_TEST_REQUIRES($KEY)" "${TEST_TAGS[@]}" ; then + echo "Test skipped due to $KEY=n (test has \"NFT_TEST_REQUIRES($KEY)\" tag)" >> "$NFT_TEST_TESTTMPDIR/testout.log" + rc_test=77 + break + fi + done +fi + +if [ "$rc_test" -eq 0 ] ; then + for KEY in $(compgen -v | grep '^NFT_TEST_SKIP_') ; do + if [ "${!KEY}" != y ]; then + continue + fi + ensure_TEST_TAGS "$TEST" + if array_contains "NFT_TEST_SKIP($KEY)" "${TEST_TAGS[@]}" ; then + echo "Test skipped due to $KEY=y (test has \"NFT_TEST_SKIP($KEY)\" tag)" >> "$NFT_TEST_TESTTMPDIR/testout.log" + rc_test=77 + break + fi + done +fi + +if [ "$rc_test" -eq 0 ] ; then + CMD=( "$TEST" ) + if [ "$NFT_TEST_VERBOSE_TEST" = y ] ; then + X="$(sed -n '1 s/^#!\(\/bin\/bash\>.*$\)/\1/p' "$TEST" 2>/dev/null)" + if [ -n "$X" ] ; then + # Note that kernel parses the shebang differently and does not + # word splitting for the arguments. We do split the arguments here + # which would matter if there are spaces. For our tests, there + # are either no arguments or only one argument without space. So + # this is good enough. + CMD=( $X -x "$TEST" ) + fi + fi + printf "Command: $(printf '%q ' "${CMD[@]}")\n" &>> "$NFT_TEST_TESTTMPDIR/testout.log" + "${CMD[@]}" &>> "$NFT_TEST_TESTTMPDIR/testout.log" || rc_test=$? +fi + +rc_chkdump=0 +rc=0 +$NFT list ruleset > "$NFT_TEST_TESTTMPDIR/ruleset-after" 2> "$NFT_TEST_TESTTMPDIR/chkdump" || rc=$? +if [ "$rc" -ne 0 -o -s "$NFT_TEST_TESTTMPDIR/chkdump" ] ; then + show_file "$NFT_TEST_TESTTMPDIR/chkdump" "Command \`$NFT list ruleset\` failed" >> "$NFT_TEST_TESTTMPDIR/rc-failed-chkdump" + rc_chkdump=1 +fi +if [ "$NFT_TEST_HAVE_json" != n ] ; then + rc=0 + $NFT -j list ruleset > "$NFT_TEST_TESTTMPDIR/ruleset-after.json" 2> "$NFT_TEST_TESTTMPDIR/chkdump" || rc=$? + + # Workaround known bug in stmt_print_json(), due to + # "chain_stmt_ops.json" being NULL. This spams stderr. + sed -i '/^warning: stmt ops chain have no json callback$/d' "$NFT_TEST_TESTTMPDIR/chkdump" + + if [ "$rc" -ne 0 -o -s "$NFT_TEST_TESTTMPDIR/chkdump" ] ; then + show_file "$NFT_TEST_TESTTMPDIR/chkdump" "Command \`$NFT -j list ruleset\` failed" >> "$NFT_TEST_TESTTMPDIR/rc-failed-chkdump" + rc_chkdump=1 + fi + # JSON output needs normalization/sanitization, otherwise it's not stable. + "$NFT_TEST_BASEDIR/helpers/json-sanitize-ruleset.sh" "$NFT_TEST_TESTTMPDIR/ruleset-after.json" + json_pretty "$NFT_TEST_TESTTMPDIR/ruleset-after.json" > "$NFT_TEST_TESTTMPDIR/ruleset-after.json-pretty" +fi + +read tainted_after < /proc/sys/kernel/tainted + +DUMPPATH="$TESTDIR/dumps" +DUMPFILE="$DUMPPATH/$TESTBASE.nft" +JDUMPFILE="$DUMPPATH/$TESTBASE.json-nft" +NODUMPFILE="$DUMPPATH/$TESTBASE.nodump" + +# The caller can request a re-geneating of the .nft, .nodump, .json-nft dump files +# by setting DUMPGEN=y. In that case, only the existing files will be regenerated +# (unless all three files are missing, in which case all of them are generated). +# +# By setting DUMPGEN=all, all 3 files are always regenerated. +dump_written=n +if [ "$rc_test" -eq 0 -a '(' "$DUMPGEN" = all -o "$DUMPGEN" = y ')' ] ; then + dump_written=y + if [ ! -d "$DUMPPATH" ] ; then + mkdir "$DUMPPATH" + fi + if [ "$DUMPGEN" = all ] ; then + gen_nodumpfile=y + gen_dumpfile=y + gen_jdumpfile=y + else + # by default, only regenerate the files that we already have on disk. + gen_nodumpfile=n + gen_dumpfile=n + gen_jdumpfile=n + test -f "$DUMPFILE" && gen_dumpfile=y + test -f "$JDUMPFILE" && gen_jdumpfile=y + test -f "$NODUMPFILE" && gen_nodumpfile=y + if [ "$gen_dumpfile" != y -a "$gen_jdumpfile" != y -a "$gen_nodumpfile" != y ] ; then + # Except, if no files exist. Them generate all files. + gen_dumpfile=y + gen_jdumpfile=y + gen_nodumpfile=y + fi + fi + if [ "$gen_nodumpfile" = y ] ; then + : > "$NODUMPFILE" + fi + if [ "$gen_dumpfile" = y ] ; then + cat "$NFT_TEST_TESTTMPDIR/ruleset-after" > "$DUMPFILE" + fi + if [ "$NFT_TEST_HAVE_json" != n -a "$gen_jdumpfile" = y ] ; then + cat "$NFT_TEST_TESTTMPDIR/ruleset-after.json-pretty" > "$JDUMPFILE" + fi +fi + +rc_dump=0 +if [ "$rc_test" -ne 77 -a "$dump_written" != y ] ; then + if [ -f "$DUMPFILE" ] ; then + if ! $DIFF -u "$DUMPFILE" "$NFT_TEST_TESTTMPDIR/ruleset-after" &> "$NFT_TEST_TESTTMPDIR/ruleset-diff" ; then + show_file "$NFT_TEST_TESTTMPDIR/ruleset-diff" "Failed \`$DIFF -u \"$DUMPFILE\" \"$NFT_TEST_TESTTMPDIR/ruleset-after\"\`" >> "$NFT_TEST_TESTTMPDIR/rc-failed-dump" + rc_dump=1 + else + rm -f "$NFT_TEST_TESTTMPDIR/ruleset-diff" + fi + fi + if [ "$NFT_TEST_HAVE_json" != n -a -f "$JDUMPFILE" ] ; then + if ! $DIFF -u "$JDUMPFILE" "$NFT_TEST_TESTTMPDIR/ruleset-after.json-pretty" &> "$NFT_TEST_TESTTMPDIR/ruleset-diff.json" ; then + show_file "$NFT_TEST_TESTTMPDIR/ruleset-diff.json" "Failed \`$DIFF -u \"$JDUMPFILE\" \"$NFT_TEST_TESTTMPDIR/ruleset-after.json-pretty\"\`" >> "$NFT_TEST_TESTTMPDIR/rc-failed-dump" + rc_dump=1 + else + rm -f "$NFT_TEST_TESTTMPDIR/ruleset-diff.json" + fi + fi +fi + +# check that a flush after the test succeeds. We anyway need a clean ruleset +# for the `nft --check` next. +rc=0 +$NFT flush ruleset &> "$NFT_TEST_TESTTMPDIR/chkdump" || rc=1 +if [ "$rc" = 1 -o -s "$NFT_TEST_TESTTMPDIR/chkdump" ] ; then + show_file "$NFT_TEST_TESTTMPDIR/chkdump" "Command \`$NFT flush ruleset\` failed" >> "$NFT_TEST_TESTTMPDIR/rc-failed-chkdump" + rc_chkdump=1 +fi +# Check that `nft [-j] list ruleset | nft [-j] --check -f -` works. +fail=n +$NFT --check -f "$NFT_TEST_TESTTMPDIR/ruleset-after" &> "$NFT_TEST_TESTTMPDIR/chkdump" || fail=y +test -s "$NFT_TEST_TESTTMPDIR/chkdump" && fail=y +if [ "$fail" = y ] ; then + show_file "$NFT_TEST_TESTTMPDIR/chkdump" "Command \`$NFT --check -f \"$NFT_TEST_TESTTMPDIR/ruleset-after\"\` failed" >> "$NFT_TEST_TESTTMPDIR/rc-failed-chkdump" + rc_chkdump=1 +fi +if [ -f "$DUMPFILE" ] && ! cmp "$DUMPFILE" "$NFT_TEST_TESTTMPDIR/ruleset-after" &>/dev/null ; then + # Also check the $DUMPFILE to hit possibly new code paths. This + # is useful to see crashes and with ASAN/valgrind. + $NFT --check -f "$DUMPFILE" &>/dev/null || : +fi +if [ "$NFT_TEST_HAVE_json" != n ] ; then + if [ ! -f "$JDUMPFILE" ] ; then + # Optimally, `nft -j list ruleset | nft -j --check -f -` never + # fails. However, there are known issues where this doesn't + # work, and we cannot assert hard against that. It's those + # tests that don't have a .json-nft file. + # + # This should be fixed, every test should have a .json-nft + # file, and this workaround removed. + $NFT -j --check -f "$NFT_TEST_TESTTMPDIR/ruleset-after.json" &>/dev/null || : + $NFT -j --check -f "$NFT_TEST_TESTTMPDIR/ruleset-after.json-pretty" &>/dev/null || : + else + fail=n + $NFT -j --check -f "$NFT_TEST_TESTTMPDIR/ruleset-after.json" &> "$NFT_TEST_TESTTMPDIR/chkdump" || fail=y + test -s "$NFT_TEST_TESTTMPDIR/chkdump" && fail=y + if [ "$fail" = y ] ; then + show_file "$NFT_TEST_TESTTMPDIR/chkdump" "Command \`$NFT -j --check -f \"$NFT_TEST_TESTTMPDIR/ruleset-after.json\"\` failed" >> "$NFT_TEST_TESTTMPDIR/rc-failed-chkdump" + rc_chkdump=1 + fi + fail=n + $NFT -j --check -f "$NFT_TEST_TESTTMPDIR/ruleset-after.json-pretty" &> "$NFT_TEST_TESTTMPDIR/chkdump" || fail=y + test -s "$NFT_TEST_TESTTMPDIR/chkdump" && fail=y + if [ "$fail" = y ] ; then + show_file "$NFT_TEST_TESTTMPDIR/chkdump" "Command \`$NFT -j --check -f \"$NFT_TEST_TESTTMPDIR/ruleset-after.json-pretty\"\` failed" >> "$NFT_TEST_TESTTMPDIR/rc-failed-chkdump" + rc_chkdump=1 + fi + fi + if [ -f "$JDUMPFILE" ] \ + && ! cmp "$JDUMPFILE" "$NFT_TEST_TESTTMPDIR/ruleset-after.json" &>/dev/null \ + && ! cmp "$JDUMPFILE" "$NFT_TEST_TESTTMPDIR/ruleset-after.json-pretty" &>/dev/null ; \ + then + $NFT -j --check -f "$JDUMPFILE" &>/dev/null || : + fi +fi +rm -f "$NFT_TEST_TESTTMPDIR/chkdump" + +rc_valgrind=0 +[ -f "$NFT_TEST_TESTTMPDIR/rc-failed-valgrind" ] && rc_valgrind=1 + +rc_tainted=0 +if [ "$tainted_before" != "$tainted_after" ] ; then + echo "$tainted_after" > "$NFT_TEST_TESTTMPDIR/rc-failed-tainted" + rc_tainted=1 +fi + +if [ "$rc_valgrind" -ne 0 ] ; then + rc_exit=122 +elif [ "$rc_tainted" -ne 0 ] ; then + rc_exit=123 +elif [ "$rc_test" -ge 118 -a "$rc_test" -le 124 ] ; then + # Special exit codes are reserved. Coerce them. + rc_exit=125 +elif [ "$rc_test" -ne 0 ] ; then + rc_exit="$rc_test" +elif [ "$rc_dump" -ne 0 ] ; then + rc_exit=124 +elif [ "$rc_chkdump" -ne 0 ] ; then + rc_exit=121 +else + rc_exit=0 +fi + + +# We always write the real exit code of the test ($rc_test) to one of the files +# rc-{ok,skipped,failed}, depending on which it is. +# +# Note that there might be other rc-failed-{dump,tainted,valgrind} files with +# additional errors. Note that if such files exist, the overall state will +# always be failed too (and an "rc-failed" file exists). +# +# On failure, we also write the combined "$rc_exit" code from "test-wrapper.sh" +# to "rc-failed-exit" file. +# +# This means, failed tests will have a "rc-failed" file, and additional +# "rc-failed-*" files exist for further information. +if [ "$rc_exit" -eq 0 ] ; then + RC_FILENAME="rc-ok" +elif [ "$rc_exit" -eq 77 ] ; then + RC_FILENAME="rc-skipped" +else + RC_FILENAME="rc-failed" + echo "$rc_exit" > "$NFT_TEST_TESTTMPDIR/rc-failed-exit" +fi +echo "$rc_test" > "$NFT_TEST_TESTTMPDIR/$RC_FILENAME" + +END_TIME="$(cut -d ' ' -f1 /proc/uptime)" +WALL_TIME="$(awk -v start="$START_TIME" -v end="$END_TIME" "BEGIN { print(end - start) }")" +printf "%s\n" "$WALL_TIME" "$START_TIME" "$END_TIME" > "$NFT_TEST_TESTTMPDIR/times" + +exit "$rc_exit" diff --git a/tests/shell/run-tests.sh b/tests/shell/run-tests.sh index 931bba967..2d2e0ad14 100755 --- a/tests/shell/run-tests.sh +++ b/tests/shell/run-tests.sh @@ -1,57 +1,617 @@ #!/bin/bash -# Configuration -TESTDIR="./$(dirname $0)/testcases" -SRC_NFT="$(dirname $0)/../../src/nft" -DIFF=$(which diff) +unset LANGUAGE +export LANG=C +export LC_ALL=C + +GREEN="" +YELLOW="" +RED="" +RESET="" +if [ -z "$NO_COLOR" ] ; then + if [ -n "$CLICOLOR_FORCE" ] || [[ -t 1 ]] ; then + # See https://bixense.com/clicolors/ . We only check isatty() on + # file descriptor 1, to decide whether colorizing happens (although, + # we might also colorize on other places/FDs). + GREEN=$'\e[32m' + YELLOW=$'\e[33m' + RED=$'\e[31m' + RESET=$'\e[0m' + fi +fi + +array_contains() { + local needle="$1" + local a + shift + for a; do + [ "$a" = "$needle" ] && return 0 + done + return 1 +} + +array_remove_first() { + local _varname="$1" + local _needle="$2" + local _result=() + local _a + + eval "local _input=( \"\${$_varname[@]}\" )" + for _a in "${_input[@]}" ; do + if [ -n "${_needle+x}" -a "$_needle" = "$_a" ] ; then + unset _needle + else + _result+=("$_a") + fi + done + eval "$_varname="'( "${_result[@]}" )' +} + +colorize_keywords() { + local out_variable="$1" + local color="$2" + local val="$3" + local val2 + shift 3 + + printf -v val2 '%q' "$val" + array_contains "$val" "$@" && val2="$color$val2$RESET" + printf -v "$out_variable" '%s' "$val2" +} + +strtonum() { + local s="$1" + local n + local n2 + + re='^[[:space:]]*([0-9]+)[[:space:]]*$' + if [[ "$s" =~ $re ]] ; then + n="${BASH_REMATCH[1]}" + if [ "$(( n + 0 ))" = "$n" ] ; then + echo "$n" + return 0 + fi + fi + re='^[[:space:]]*0x([0-9a-fA-F]+)[[:space:]]*$' + if [[ "$s" =~ $re ]] ; then + n="${BASH_REMATCH[1]}" + n2="$(( 16#$n + 0 ))" + if [ "$n2" = "$(printf '%d' "0x$n" 2>/dev/null)" ] ; then + echo "$n2" + return 0 + fi + fi + return 1 +} + +_msg() { + local level="$1" + shift + + if [ "$level" = E ] ; then + printf '%s\n' "$RED$level$RESET: $*" + elif [ "$level" = W ] ; then + printf '%s\n' "$YELLOW$level$RESET: $*" + else + printf '%s\n' "$level: $*" + fi + if [ "$level" = E ] ; then + exit 1 + fi +} msg_error() { - echo "E: $1 ..." >&2 - exit 1 + _msg E "$@" } msg_warn() { - echo "W: $1" >&2 + _msg W "$@" } msg_info() { - echo "I: $1" + _msg I "$@" } -if [ "$(id -u)" != "0" ] ; then - msg_error "this requires root!" +align_text() { + local _OUT_VARNAME="$1" + local _LEFT_OR_RIGHT="$2" + local _INDENT="$3" + shift 3 + local _text="$*" + local _text_plain + local _text_align + local _text_result + local _i + + # This function is needed, because "$text" might contain color escape + # sequences. A plain `printf '%12s' "$text"` will not align properly. + + # strip escape sequences + _text_plain="${_text//$'\e['[0-9]m/}" + _text_plain="${_text_plain//$'\e['[0-9][0-9]m/}" + + _text_align="" + for (( _i = "${#_text_plain}" ; "$_i" < "$_INDENT" ; _i++ )) ; do + _text_align="$_text_align " + done + + if [ "$_LEFT_OR_RIGHT" = left ] ; then + _text_result="$(printf "%s$_text_align-" "$_text")" + else + _text_result="$(printf "$_text_align%s-" "$_text")" + fi + _text_result="${_text_result%-}" + + eval "$_OUT_VARNAME=\"\$_text_result\"" +} + +bool_n() { + case "$1" in + n|N|no|No|NO|0|false|False|FALSE) + printf n + ;; + *) + printf y + ;; + esac +} + +bool_y() { + case "$1" in + y|Y|yes|Yes|YES|1|true|True|TRUE) + printf y + ;; + *) + printf n + ;; + esac +} + +usage() { + echo " $0 [OPTIONS] [TESTS...]" + echo + echo "OPTIONS:" + echo " -h|--help : Print usage." + echo " -L|--list-tests : List test names and quit." + echo " -v : Sets VERBOSE=y." + echo " -g : Sets DUMPGEN=y." + echo " -V : Sets VALGRIND=y." + echo " -K : Sets KMEMLEAK=y." + echo " -R|--without-realroot : Sets NFT_TEST_HAS_REALROOT=n." + echo " -U|--no-unshare : Sets NFT_TEST_UNSHARE_CMD=\"\"." + echo " -k|--keep-logs : Sets NFT_TEST_KEEP_LOGS=y." + echo " -x : Sets NFT_TEST_VERBOSE_TEST=y." + echo " -s|--sequential : Sets NFT_TEST_JOBS=0, which also enables global cleanups." + echo " Also sets NFT_TEST_SHUFFLE_TESTS=n if left unspecified." + echo " -Q|--quick : Sets NFT_TEST_SKIP_slow=y." + echo " -S|--setup-host : Modify the host to run as rootless. Otherwise, some tests will be" + echo " skipped. Basically, this bumps /proc/sys/net/core/{wmem_max,rmem_max}." + echo " Must run as root and this option must be specified alone." + echo " -- : Separate options from tests." + echo " [TESTS...] : Other options are treated as test names," + echo " that is, executables that are run by the runner." + echo + echo "ENVIRONMENT VARIABLES:" + echo " NFT= : Path to nft executable. Will be called as \`\$NFT [...]\` so" + echo " it can be a command with parameters. Note that in this mode quoting" + echo " does not work, so the usage is limited and the command cannot contain" + echo " spaces." + echo " NFT_REAL= : Real nft comand. Usually this is just the same as \$NFT," + echo " however, you may set NFT='valgrind nft' and NFT_REAL to the real command." + echo " VERBOSE=*|y : Enable verbose output." + echo " NFT_TEST_VERBOSE_TEST=*|y: if true, enable verbose output for tests. For bash scripts, this means" + echo " to pass \"-x\" to the interpreter." + echo " DUMPGEN=*|y|all : Regenerate dump files \".{nft,json-nft,nodump}\". \"DUMPGEN=y\" only regenerates existing" + echo " files, unless the test has no files (then all three files are generated, and you need to" + echo " choose which to keep). With \"DUMPGEN=all\" all 3 files are regenerated, regardless" + echo " whether they already exist." + echo " VALGRIND=*|y : Run \$NFT in valgrind." + echo " KMEMLEAK=*|y : Check for kernel memleaks." + echo " NFT_TEST_HAS_REALROOT=*|y : To indicate whether the test has real root permissions." + echo " Usually, you don't need this and it gets autodetected." + echo " You might want to set it, if you know better than the" + echo " \`id -u\` check, whether the user is root in the main namespace." + echo " Note that without real root, certain tests may not work," + echo " e.g. due to limited /proc/sys/net/core/{wmem_max,rmem_max}." + echo " Checks that cannot pass in such environment should check for" + echo " [ \"\$NFT_TEST_HAS_REALROOT\" != y ] and skip gracefully." + echo " NFT_TEST_HAS_SOCKET_LIMITS=*|n : some tests will fail if /proc/sys/net/core/{wmem_max,rmem_max} is" + echo " too small. When running as real root, then test can override those limits. However," + echo " with rootless the test would fail. Tests will check for [ "\$NFT_TEST_HAS_SOCKET_LIMITS" = y ]" + echo " and skip. You may set NFT_TEST_HAS_SOCKET_LIMITS=n if you ensure those limits are" + echo " suitable to run the test rootless. Otherwise will be autodetected." + echo " Set /proc/sys/net/core/{wmem_max,rmem_max} to at least 4MB to get them to pass automatically." + echo " NFT_TEST_UNSHARE_CMD=cmd : when set, this is the command line for an unshare" + echo " command, which is used to sandbox each test invocation. By" + echo " setting it to empty, no unsharing is done." + echo " By default it is unset, in which case it's autodetected as" + echo " \`unshare -f -p\` (for root) or as \`unshare -f -p --mount-proc -U --map-root-user -n\`" + echo " for non-root." + echo " When setting this, you may also want to set NFT_TEST_HAS_UNSHARED=," + echo " NFT_TEST_HAS_REALROOT= and NFT_TEST_HAS_UNSHARED_MOUNT= accordingly." + echo " NFT_TEST_HAS_UNSHARED=*|y : To indicate to the test whether the test run will be unshared." + echo " Test may consider this." + echo " This is only honored when \$NFT_TEST_UNSHARE_CMD= is set. Otherwise it's detected." + echo " NFT_TEST_HAS_UNSHARED_MOUNT=*|y : To indicate to the test whether the test run will have a private" + echo " mount namespace." + echo " This is only honored when \$NFT_TEST_UNSHARE_CMD= is set. Otherwise it's detected." + echo " NFT_TEST_KEEP_LOGS=*|y: Keep the temp directory. On success, it will be deleted by default." + echo " NFT_TEST_JOBS=: number of jobs for parallel execution. Defaults to \"\$(nproc)*1.5\" for parallel run." + echo " Setting this to \"0\" or \"1\", means to run jobs sequentially." + echo " Setting this to \"0\" means also to perform global cleanups between tests (remove" + echo " kernel modules)." + echo " Parallel jobs requires unshare and are disabled with NFT_TEST_UNSHARE_CMD=\"\"." + echo " NFT_TEST_FAIL_ON_SKIP=*|y: if any jobs are skipped, exit with error." + echo " NFT_TEST_RANDOM_SEED=: The test runner will export the environment variable NFT_TEST_RANDOM_SEED" + echo " set to a random number. This can be used as a stable seed for tests to randomize behavior." + echo " Set this to a fixed value to get reproducible behavior." + echo " NFT_TEST_SHUFFLE_TESTS=*|n|y: control whether to randomly shuffle the order of tests. By default, if" + echo " tests are specified explicitly, they are not shuffled while they are shuffled when" + echo " all tests are run. The shuffling is based on NFT_TEST_RANDOM_SEED." + echo " TMPDIR= : select a different base directory for the result data." + echo + echo " NFT_TEST_HAVE_=*|y: Some tests requires certain features or will be skipped." + echo " The features are autodetected, but you can force it by setting the variable." + echo " Supported s are: ${_HAVE_OPTS[@]}." + echo " NFT_TEST_SKIP_