tests: adapt test suite to run with legacy+nftables based binaries

While at it, make following changes/fixes:
 1. run each test in a fresh net namespace
 2. remove rmmod use, its very distuptive and not needed after 1.
 3. avoid -e use if possible
 4. make sure we exit 0 when test is expected to fail
 5. set XT_LIBDIR so we point at the correct extensions to be used

Also delete 0003duplicate_1, its same test as 0001duplicate_1.

NB: I don't think its good to have this 'encode retval in name' scheme.

These are scripts, so they should always return 0, i.e. do

  iptables --this-command-should-fail || exit 0
  echo "succeess, should fail"
  exit 1

Much simpler, imo.  This was inherited from nft shell tests
though and changing it there is rather intrusive so use same scheme for
now.

Signed-off-by: Florian Westphal <fw@strlen.de>

diff --git a/iptables/tests/shell/run-tests.sh b/iptables/tests/shell/run-tests.sh
index cf5cbdc30cf942579b0b100dca3a171cbbd11e65..38f3c2d491248cc012d751b8dc7f2c84747297e4 100755 (executable)
--- a/iptables/tests/shell/run-tests.sh
+++ b/iptables/tests/shell/run-tests.sh
@@ -3,8 +3,10 @@
 #configuration
 TESTDIR="./$(dirname $0)/"
 RETURNCODE_SEPARATOR="_"
-XTABLES_MULTI="$(dirname $0)/../../xtables-multi"
-DIFF=$(which diff)
+XTABLES_NFT_MULTI="$(dirname $0)/../../xtables-nft-multi"
+XTABLES_LEGACY_MULTI="$(dirname $0)/../../xtables-legacy-multi"
+
+export XTABLES_LIBDIR=${TESTDIR}/../../../extensions
 
 msg_error() {
         echo "E: $1 ..." >&2
@@ -23,32 +25,10 @@ if [ "$(id -u)" != "0" ] ; then
         msg_error "this requires root!"
 fi
 
-[ -z "$IPTABLES" ] && IPTABLES=$XTABLES_MULTI
-if [ ! -x "$IPTABLES" ] ; then
-        msg_error "no xtables-multi binary!"
-else
-        msg_info "using xtables-multi binary $IPTABLES"
-fi
-
 if [ ! -d "$TESTDIR" ] ; then
         msg_error "missing testdir $TESTDIR"
 fi
 
-FIND="$(which find)"
-if [ ! -x "$FIND" ] ; then
-        msg_error "no find binary found"
-fi
-
-MODPROBE="$(which modprobe)"
-if [ ! -x "$MODPROBE" ] ; then
-        msg_error "no modprobe binary found"
-fi
-
-DEPMOD="$(which depmod)"
-if [ ! -x "$DEPMOD" ] ; then
-        msg_error "no depmod binary found"
-fi
-
 if [ "$1" == "-v" ] ; then
         VERBOSE=y
         shift
@@ -63,67 +43,65 @@ for arg in "$@"; do
         fi
 done
 
-kernel_cleanup() {
-       for it in iptables ip6tables; do
-       for table in filter mangle nat raw; do
-               $it -t $table -nL >/dev/null 2>&1 || continue # non-existing table
-               $it -t $table -F        # delete rules
-               $it -t $table -X        # delete custom chains
-               $it -t $table -Z        # zero counters
-       done
-       done
-       $DEPMOD -a
-       $MODPROBE -raq \
-       ip_tables iptable_nat iptable_mangle ipt_REJECT
-}
-
 find_tests() {
         if [ ! -z "$SINGLE" ] ; then
                 echo $SINGLE
                 return
         fi
-        ${FIND} ${TESTDIR} -executable -regex \
+        find ${TESTDIR} -executable -regex \
                 .*${RETURNCODE_SEPARATOR}[0-9]+ | sort
 }
 
-
-echo ""
 ok=0
 failed=0
 
-for testfile in $(find_tests)
-do
+do_test() {
+       testfile="$1"
+       xtables_multi="$2"
 
        for it in iptables ip6tables; do
-               kernel_cleanup
                rc_spec=`echo $(basename ${testfile}) | cut -d _ -f2-`
-               IPTABLES="$XTABLES_MULTI $it"
+               IPTABLES="$xtables_multi $it"
 
                msg_info "[EXECUTING]   $testfile"
-               test_output=$(IPTABLES=$IPTABLES ${testfile} 2>&1)
+
+               if [ "$VERBOSE" = "y" ]; then
+                       IPTABLES="$IPTABLES" unshare -n ${testfile}
+               else
+                       IPTABLES="$IPTABLES" unshare -n ${testfile} > /dev/null 2>&1
+               fi
+
                rc_got=$?
                echo -en "\033[1A\033[K" # clean the [EXECUTING] foobar line
 
                if [ "$rc_got" == "$rc_spec" ] ; then
                        msg_info "[OK]          $testfile"
-                       [ "$VERBOSE" == "y" ] && [ ! -z "$test_output" ] && echo "$test_output"
                        ((ok++))
-
                else
                        ((failed++))
-                       if [ "$VERBOSE" == "y" ] ; then
-                               msg_warn "[FAILED]      $testfile: expected $rc_spec but got $rc_got"
-                               [ ! -z "$test_output" ] && echo "$test_output"
-                       else
-                               msg_warn "[FAILED]      $testfile"
-                       fi
+                       msg_warn "[FAILED]      $testfile: expected $rc_spec but got $rc_got"
                fi
-
        done
-done
+}
 
 echo ""
-msg_info "results: [OK] $ok [FAILED] $failed [TOTAL] $((ok+failed))"
+for testfile in $(find_tests);do
+       do_test "$testfile" "$XTABLES_LEGACY_MULTI"
+done
+msg_info "legacy results: [OK] $ok [FAILED] $failed [TOTAL] $((ok+failed))"
+
+legacy_ok=$ok
+legacy_fail=$failed
+ok=0
+failed=0
+for testfile in $(find_tests);do
+       do_test "$testfile" "$XTABLES_NFT_MULTI"
+done
+msg_info "nft results: [OK] $ok [FAILED] $failed [TOTAL] $((ok+failed))"
+
+ok=$((legacy_ok+ok))
+failed=$((legacy_fail+failed))
+
+msg_info "combined results: [OK] $ok [FAILED] $failed [TOTAL] $((ok+failed))"
 
-kernel_cleanup
 exit 0

diff --git a/iptables/tests/shell/testcases/chain/0001duplicate_1 b/iptables/tests/shell/testcases/chain/0001duplicate_1
index 6d42cecf2de55bc38b192cd0a4672327dca36aad..e88f82ad24ff7e19db74430f8c0d8682e326dd76 100755 (executable)
--- a/iptables/tests/shell/testcases/chain/0001duplicate_1
+++ b/iptables/tests/shell/testcases/chain/0001duplicate_1
@@ -1,11 +1,9 @@
 #!/bin/bash
 
-set -e
+set -x
 
 $IPTABLES -t filter -N c1
-$IPTABLES -t filter -N c1
+$IPTABLES -t filter -N c1 || exit 1
 
-if [ $? -eq 0 ]; then
-       echo "E: Duplicate chains" >&2
-       exit 0
-fi
+echo "E: Duplicate chains" >&2
+exit 0

diff --git a/iptables/tests/shell/testcases/chain/0002duplicate_0 b/iptables/tests/shell/testcases/chain/0002duplicate_0
deleted file mode 100755 (executable)
index 025114e..0000000
--- a/iptables/tests/shell/testcases/chain/0002duplicate_0
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-$IPTABLES -t filter -N c1 || exit 1
-$IPTABLES -t filter -N c1
-
-if [ $? -eq 0 ]; then
-       echo "E: Duplicate chains" >&2
-       exit 0
-fi

diff --git a/iptables/tests/shell/testcases/chain/0003duplicate_1 b/iptables/tests/shell/testcases/chain/0003duplicate_1
deleted file mode 100755 (executable)
index 6d42cec..0000000
--- a/iptables/tests/shell/testcases/chain/0003duplicate_1
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -e
-
-$IPTABLES -t filter -N c1
-$IPTABLES -t filter -N c1
-
-if [ $? -eq 0 ]; then
-       echo "E: Duplicate chains" >&2
-       exit 0
-fi

diff --git a/iptables/tests/shell/testcases/chain/0005rename_1 b/iptables/tests/shell/testcases/chain/0005rename_1
index 7261b6dcf2b233049e53a24762b8b8ee041ec631..f2f6e55af44b232d9b93ccba258d6073a44e54b2 100755 (executable)
--- a/iptables/tests/shell/testcases/chain/0005rename_1
+++ b/iptables/tests/shell/testcases/chain/0005rename_1
@@ -1,12 +1,8 @@
 #!/bin/bash
 
-set -e
+$IPTABLES -N c1 || exit 0
+$IPTABLES -N c2 || exit 0
+$IPTABLES -E c1 c2 || exit 1
 
-$IPTABLES -N c1
-$IPTABLES -N c2
-$IPTABLES -E c1 c2
-
-if [ $? -eq 0 ] ; then
-        echo "E: Renamed with existing chain" >&2
-        exit 0
-fi
+echo "E: Renamed with existing chain" >&2
+exit 0