testing: Make per-CPU SAs scenarios more predictable by pinning IRQs to vCPUs

This ensures that packets on sun are processed on a particular CPU and
not randomly on one, which causes expected SAs not to get created or
other weird things.

diff --git a/testing/hosts/default/usr/local/bin/collect-results b/testing/hosts/default/usr/local/bin/collect-results
index cba00d771420c229b25808aba9e888bf62bfc5cc..e89ecf993fe60fb5fcb0474e5328fb7c2a077e95 100755 (executable)
--- a/testing/hosts/default/usr/local/bin/collect-results
+++ b/testing/hosts/default/usr/local/bin/collect-results
@@ -1,4 +1,4 @@
-#! /bin/bash
+#!/bin/bash
 
 while getopts "46idt:lr" opt
 do

diff --git a/testing/hosts/default/usr/local/bin/set-cpu-affinity b/testing/hosts/default/usr/local/bin/set-cpu-affinity
new file mode 100755 (executable)
index 0000000..1c56669
--- /dev/null
+++ b/testing/hosts/default/usr/local/bin/set-cpu-affinity
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Sets the CPU affiniy of the IRQ handling inbound messages on the given
+# network interface
+
+IFACE=$1
+AFFINITY=$2
+
+if [ -z "$IFACE" -o -z "$AFFINITY" ]
+then
+       exit 1
+fi
+
+PCI_ADDR=$(ethtool -i ${IFACE} | grep bus-info | sed -e 's/bus-info: //')
+if [ -z "$PCI_ADDR" ]
+then
+       exit 2
+fi
+
+IRQ=$(grep "MSIX-${PCI_ADDR}.*input" /proc/interrupts | awk -F ':' '{print $1}' | awk '{print $1}')
+if [ -z "$IRQ" ]
+then
+       exit 3
+fi
+
+PREV=$(cat /proc/irq/${IRQ}/smp_affinity_list)
+echo "Setting affinity for IRQ ${IRQ} of ${IFACE}@${PCI_ADDR} to ${AFFINITY}, was ${PREV}"
+echo ${AFFINITY} > /proc/irq/${IRQ}/smp_affinity_list

diff --git a/testing/tests/ikev2/per-cpu-sas-encap-transport/evaltest.dat b/testing/tests/ikev2/per-cpu-sas-encap-transport/evaltest.dat
index e4ea4cfaa3c596dabe869b62ab5ec1d98e5e3221..914d53ebe5861995824706ee81fa127e6e024e8e 100755 (executable)
--- a/testing/tests/ikev2/per-cpu-sas-encap-transport/evaltest.dat
+++ b/testing/tests/ikev2/per-cpu-sas-encap-transport/evaltest.dat
@@ -31,8 +31,8 @@ moon::taskset -c 2 ping -c 1 -I PH_IP_MOON PH_IP_SUN::64 bytes from PH_IP_SUN: i
 moon::cat /var/log/daemon.log::creating acquire job for policy PH_IP_MOON/32\[icmp/8] === PH_IP_SUN/32\[icmp/8].*cpu {2}::YES
 moon::swanctl --list-sas --child-id 2 --raw 2> /dev/null::child-sas.*net-net.*packets-in=1.*packets-out=3::YES
 moon::swanctl --list-sas --child-id 3 --raw 2> /dev/null::child-sas.*net-net.*cpu=1.*packets-in=0.*packets-out=1::YES
-moon::swanctl --list-sas --child-id 4 --raw 2> /dev/null::child-sas.*net-net.*cpu=3.*packets-in=3.*packets-out=0::YES
-moon::swanctl --list-sas --child-id 5 --raw 2> /dev/null::child-sas.*net-net.*cpu=0.*packets-in=0.*packets-out=0::YES
+moon::swanctl --list-sas --child-id 4 --raw 2> /dev/null::child-sas.*net-net.*cpu=3.*packets-in=2.*packets-out=0::YES
+moon::swanctl --list-sas --child-id 5 --raw 2> /dev/null::child-sas.*net-net.*cpu=0.*packets-in=1.*packets-out=0::YES
 moon::swanctl --list-sas --child-id 6 --raw 2> /dev/null::child-sas.*net-net.*cpu=2.*packets-in=0.*packets-out=0::YES
 # all these pings must be sent via their respective SAs, never the fallback SA
 moon::taskset -c 0 ping -c 1 -I PH_IP_MOON PH_IP_SUN::64 bytes from PH_IP_SUN: icmp_.eq=1::YES
@@ -41,21 +41,19 @@ moon::taskset -c 2 ping -c 1 -I PH_IP_MOON PH_IP_SUN::64 bytes from PH_IP_SUN: i
 moon::taskset -c 3 ping -c 1 -I PH_IP_MOON PH_IP_SUN::64 bytes from PH_IP_SUN: icmp_.eq=1::YES
 moon::swanctl --list-sas --child-id 2 --raw 2> /dev/null::child-sas.*net-net.*packets-in=1.*packets-out=3::YES
 moon::swanctl --list-sas --child-id 3 --raw 2> /dev/null::child-sas.*net-net.*cpu=1.*packets-in=0.*packets-out=2::YES
-moon::swanctl --list-sas --child-id 4 --raw 2> /dev/null::child-sas.*net-net.*cpu=3.*packets-in=7.*packets-out=1::YES
-moon::swanctl --list-sas --child-id 5 --raw 2> /dev/null::child-sas.*net-net.*cpu=0.*packets-in=0.*packets-out=1::YES
-moon::swanctl --list-sas --child-id 6 --raw 2> /dev/null::child-sas.*net-net.*cpu=2.*packets-in=0.*packets-out=1::YES
+moon::swanctl --list-sas --child-id 4 --raw 2> /dev/null::child-sas.*net-net.*cpu=3.*packets-in=2.*packets-out=1::YES
+moon::swanctl --list-sas --child-id 5 --raw 2> /dev/null::child-sas.*net-net.*cpu=0.*packets-in=1.*packets-out=1::YES
+moon::swanctl --list-sas --child-id 6 --raw 2> /dev/null::child-sas.*net-net.*cpu=2.*packets-in=4.*packets-out=1::YES
 # rekey an SA as initiator
 moon::swanctl --rekey --child-id 5
 moon::sleep .5
 moon::taskset -c 0 ping -c 1 -I PH_IP_MOON PH_IP_SUN::64 bytes from PH_IP_SUN: icmp_.eq=1::YES
-moon::swanctl --list-sas --child-id 4 --raw 2> /dev/null::child-sas.*net-net.*cpu=3.*packets-in=8.*packets-out=1::YES
-moon::swanctl --list-sas --child-id 7 --raw 2> /dev/null::child-sas.*net-net.*cpu=0.*packets-in=0.*packets-out=1::YES
+moon::swanctl --list-sas --child-id 7 --raw 2> /dev/null::child-sas.*net-net.*cpu=0.*packets-in=1.*packets-out=1::YES
 # rekey an SA as responder
 sun::swanctl --rekey --child-id 6
 moon::sleep .5
 moon::taskset -c 2 ping -c 1 -I PH_IP_MOON PH_IP_SUN::64 bytes from PH_IP_SUN: icmp_.eq=1::YES
-moon::swanctl --list-sas --child-id 4 --raw 2> /dev/null::child-sas.*net-net.*cpu=3.*packets-in=9.*packets-out=1::YES
-moon::swanctl --list-sas --child-id 8 --raw 2> /dev/null::child-sas.*net-net.*cpu=2.*packets-in=0.*packets-out=1::YES
+moon::swanctl --list-sas --child-id 8 --raw 2> /dev/null::child-sas.*net-net.*cpu=2.*packets-in=1.*packets-out=1::YES
 # rekey the fallback SA
 moon::swanctl --rekey --child-id 2
 moon::sleep .5
@@ -68,11 +66,11 @@ moon::taskset -c 2 ping -c 1 -I PH_IP_MOON PH_IP_SUN::64 bytes from PH_IP_SUN: i
 moon::taskset -c 3 ping -c 1 -I PH_IP_MOON PH_IP_SUN::64 bytes from PH_IP_SUN: icmp_.eq=1::YES
 moon::swanctl --list-sas --child-id 2 --raw 2> /dev/null::child-sas.*net-net.*state=DELETED.*packets-in=1.*packets-out=3::YES
 moon::swanctl --list-sas --child-id 3 --raw 2> /dev/null::child-sas.*net-net.*state=INSTALLED.*cpu=1.*packets-in=0.*packets-out=3::YES
-moon::swanctl --list-sas --child-id 4 --raw 2> /dev/null::child-sas.*net-net.*state=INSTALLED.*cpu=3.*packets-in=13.*packets-out=2::YES
-moon::swanctl --list-sas --child-id 5 --raw 2> /dev/null::child-sas.*net-net.*state=DELETED.*cpu=0.*packets-in=0.*packets-out=1::YES
-moon::swanctl --list-sas --child-id 6 --raw 2> /dev/null::child-sas.*net-net.*state=DELETED.*cpu=2.*packets-in=0.*packets-out=1::YES
-moon::swanctl --list-sas --child-id 7 --raw 2> /dev/null::child-sas.*net-net.*state=INSTALLED.*cpu=0.*packets-in=0.*packets-out=2::YES
-moon::swanctl --list-sas --child-id 8 --raw 2> /dev/null::child-sas.*net-net.*state=INSTALLED.*cpu=2.*packets-in=0.*packets-out=2::YES
+moon::swanctl --list-sas --child-id 4 --raw 2> /dev/null::child-sas.*net-net.*state=INSTALLED.*cpu=3.*packets-in=2.*packets-out=2::YES
+moon::swanctl --list-sas --child-id 5 --raw 2> /dev/null::child-sas.*net-net.*state=DELETED.*cpu=0.*packets-in=1.*packets-out=1::YES
+moon::swanctl --list-sas --child-id 6 --raw 2> /dev/null::child-sas.*net-net.*state=DELETED.*cpu=2.*packets-in=4.*packets-out=1::YES
+moon::swanctl --list-sas --child-id 7 --raw 2> /dev/null::child-sas.*net-net.*state=INSTALLED.*cpu=0.*packets-in=1.*packets-out=2::YES
+moon::swanctl --list-sas --child-id 8 --raw 2> /dev/null::child-sas.*net-net.*state=INSTALLED.*cpu=2.*packets-in=5.*packets-out=2::YES
 moon::swanctl --list-sas --child-id 9 --raw 2> /dev/null::child-sas.*net-net.*state=INSTALLED.*packets-in=0.*packets-out=0::YES
 sun::tcpdump::IP moon.strongswan.org.* > sun.strongswan.org.ipsec-nat-t: UDP-encap: ESP::YES
 sun::tcpdump::IP sun.strongswan.org.* > moon.strongswan.org.ipsec-nat-t: UDP-encap: ESP::YES

diff --git a/testing/tests/ikev2/per-cpu-sas-encap-transport/posttest.dat b/testing/tests/ikev2/per-cpu-sas-encap-transport/posttest.dat
index cc6a5bff73c93228760d0888e2eeb9a15ea5b09a..38d9e690daec240be19bf360b612d84a52720a7b 100755 (executable)
--- a/testing/tests/ikev2/per-cpu-sas-encap-transport/posttest.dat
+++ b/testing/tests/ikev2/per-cpu-sas-encap-transport/posttest.dat
@@ -3,3 +3,4 @@ moon::systemctl stop strongswan
 sun::systemctl stop strongswan
 moon::iptables-restore < /etc/iptables.flush
 sun::iptables-restore < /etc/iptables.flush
+sun::set-cpu-affinity eth0 0-1

diff --git a/testing/tests/ikev2/per-cpu-sas-encap-transport/pretest.dat b/testing/tests/ikev2/per-cpu-sas-encap-transport/pretest.dat
index f3538357bc33fc1095cb022f17a2a511fbf0995a..490e39b1765d974fa8d38d1e363d75517e09da8a 100755 (executable)
--- a/testing/tests/ikev2/per-cpu-sas-encap-transport/pretest.dat
+++ b/testing/tests/ikev2/per-cpu-sas-encap-transport/pretest.dat
@@ -1,3 +1,5 @@
+# make sure inbound packets trigger IRQ on the right CPU
+sun::set-cpu-affinity eth0 0
 sun::iptables-restore < /etc/iptables.rules
 moon::iptables-restore < /etc/iptables.rules
 sun::systemctl start strongswan

diff --git a/testing/tests/ikev2/per-cpu-sas-encap/posttest.dat b/testing/tests/ikev2/per-cpu-sas-encap/posttest.dat
index cc6a5bff73c93228760d0888e2eeb9a15ea5b09a..b382ffe9d46154545881a3818a8aeb6dc0583902 100755 (executable)
--- a/testing/tests/ikev2/per-cpu-sas-encap/posttest.dat
+++ b/testing/tests/ikev2/per-cpu-sas-encap/posttest.dat
@@ -3,3 +3,4 @@ moon::systemctl stop strongswan
 sun::systemctl stop strongswan
 moon::iptables-restore < /etc/iptables.flush
 sun::iptables-restore < /etc/iptables.flush
+sun::set-cpu-affinity eth1 0-1

diff --git a/testing/tests/ikev2/per-cpu-sas-encap/pretest.dat b/testing/tests/ikev2/per-cpu-sas-encap/pretest.dat
index 594837de76d1bd3388fa4c5993feca711ad1bb57..83f5b9a00689f12e0454eba5a267cc1300f464b5 100755 (executable)
--- a/testing/tests/ikev2/per-cpu-sas-encap/pretest.dat
+++ b/testing/tests/ikev2/per-cpu-sas-encap/pretest.dat
@@ -1,3 +1,5 @@
+# make sure forwarded responses trigger IRQ on the right CPU
+sun::set-cpu-affinity eth1 0
 sun::iptables-restore < /etc/iptables.rules
 moon::iptables-restore < /etc/iptables.rules
 sun::systemctl start strongswan

diff --git a/testing/tests/ikev2/per-cpu-sas/posttest.dat b/testing/tests/ikev2/per-cpu-sas/posttest.dat
index cc6a5bff73c93228760d0888e2eeb9a15ea5b09a..b382ffe9d46154545881a3818a8aeb6dc0583902 100755 (executable)
--- a/testing/tests/ikev2/per-cpu-sas/posttest.dat
+++ b/testing/tests/ikev2/per-cpu-sas/posttest.dat
@@ -3,3 +3,4 @@ moon::systemctl stop strongswan
 sun::systemctl stop strongswan
 moon::iptables-restore < /etc/iptables.flush
 sun::iptables-restore < /etc/iptables.flush
+sun::set-cpu-affinity eth1 0-1

diff --git a/testing/tests/ikev2/per-cpu-sas/pretest.dat b/testing/tests/ikev2/per-cpu-sas/pretest.dat
index 273643c480ece8e77b15ac7b6e8858d5e26f8835..d628f0369044cd08977223805d3e232b2478a253 100755 (executable)
--- a/testing/tests/ikev2/per-cpu-sas/pretest.dat
+++ b/testing/tests/ikev2/per-cpu-sas/pretest.dat
@@ -1,3 +1,5 @@
+# make sure forwarded responses trigger IRQ on the right CPU
+sun::set-cpu-affinity eth1 0
 sun::iptables-restore < /etc/iptables.rules
 moon::iptables-restore < /etc/iptables.rules
 sun::systemctl start strongswan