new approach for killing TCP connections on IP release

(This used to be ctdb commit c33a0db29b5604966f582b1f8c5fd66760c72197)

diff --git a/ctdb/config/events.d/10.interface b/ctdb/config/events.d/10.interface
index 76face39e993f49fae33e472c154614201c3cc04..540167deda291523289cb1aa58583fedb1754000 100755 (executable)
--- a/ctdb/config/events.d/10.interface
+++ b/ctdb/config/events.d/10.interface
@@ -20,6 +20,37 @@ shift
        exit 0
 }
 
+################################################
+# kill off any TCP connections with the given IP
+kill_tcp_connections() {
+    _IP="$1"    
+    _failed=0
+    _killcount=0
+    netstat -tn |egrep "^tcp.*\s+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' | 
+    while read dest src; do
+       srcip=`echo $src | cut -d: -f1`
+       srcport=`echo $src | cut -d: -f2`
+       destip=`echo $dest | cut -d: -f1`
+       destport=`echo $dest | cut -d: -f2`
+       ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
+       _killcount=`expr $_killcount + 1`
+    done
+    [ _failed = 0 ] || {
+       echo "Failed to send killtcp control"
+       return;
+    }
+    _count=0
+    while netstat -tn | grep $_IP > /dev/null; do
+       sleep 1
+       _count=`expr $_count + 1`
+       [ $_count -gt 5 ] && {
+           echo "Timed out killing tcp connections for IP $_IP"
+           return;
+       }
+    done
+    echo "`date` killed $_killcount TCP connections to released IP $_IP"
+}
+
 case $cmd in 
      #############################
      # called when ctdbd starts up
@@ -29,11 +60,6 @@ case $cmd in
        [ -f /proc/sys/net/ipv4/conf/all/arp_filter ] && {
            echo 1 > /proc/sys/net/ipv4/conf/all/arp_filter
        }
-       # make sure we don't reply to arp requests for IPs we have moved to scope
-       # host on loopback
-       [ -f /proc/sys/net/ipv4/conf/all/arp_ignore ] && {
-           echo 3 > /proc/sys/net/ipv4/conf/all/arp_ignore
-       }
        ;;
 
 
@@ -53,11 +79,12 @@ case $cmd in
                 echo "`/bin/date` Failed to bringup interface $iface"
                 exit 1
        }
-       /sbin/ip addr del $ip/32 dev lo >/dev/null 2>/dev/null
        /sbin/ip addr add $ip/$maskbits dev $iface || {
                 echo "`/bin/date` Failed to add $ip/$maskbits on dev $iface"
                 exit 1
        }
+       # cope with the script being killed while we have the interface blocked
+       /sbin/iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
 
        # flush our route cache
        echo 1 > /proc/sys/net/ipv4/route/flush
@@ -71,16 +98,32 @@ case $cmd in
           echo "`/bin/date` must supply interface, IP and maskbits"
           exit 1
        fi
+
+       # releasing an IP is a bit more complex than it seems. Once the IP
+       # is released, any open tcp connections to that IP on this host will end
+       # up being stuck. Some of them (such as NFS connections) will be unkillable
+       # so we need to use the killtcp ctdb function to kill them off. We also
+       # need to make sure that no new connections get established while we are 
+       # doing this! So what we do is this:
+       # 1) firewall this IP, so no new external packets arrive for it
+       # 2) use netstat -tn to find existing connections, and kill them 
+       # 3) remove the IP from the interface
+       # 4) remove the firewall rule
        iface=$1
        ip=$2
        maskbits=$3
-       /sbin/ip addr del $ip/$maskbits dev $iface || {
+
+       failed=0
+       # we do an extra delete to cope with the script being killed
+       /sbin/iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
+       /sbin/iptables -I INPUT -i $iface -d $ip -j DROP
+       kill_tcp_connections $ip
+       /sbin/ip addr del $ip/$maskbits dev $iface || failed=1
+       /sbin/iptables -D INPUT -i $iface -d $ip -j DROP
+       [ $failed = 0 ] || {
                 echo "`/bin/date` Failed to del $ip on dev $iface"
                 exit 1
        }
-       # we put the IP on loopback so our killtcp code can work, this allows
-       # us to avoid restarting the NFS server when we release an IP
-       /sbin/ip addr add $ip/32 dev lo scope host >/dev/null 2>/dev/null
 
        # flush our route cache
        echo 1 > /proc/sys/net/ipv4/route/flush

diff --git a/ctdb/config/events.d/60.nfs b/ctdb/config/events.d/60.nfs
index 8c9e7ebd780ed60502c1ead557fd908ebdb2e14f..f19dfea14945aab751535bdb951335b7f4ba87c9 100755 (executable)
--- a/ctdb/config/events.d/60.nfs
+++ b/ctdb/config/events.d/60.nfs
@@ -55,37 +55,7 @@ case $cmd in
        maskbits=$3
 
        echo $ip >> /etc/ctdb/state/statd/restart
-
        /bin/rm -f /etc/ctdb/state/statd/ip/$ip
-
-       # RST all tcp connections to the lockmanager
-       [ ! -z "$LOCKD_TCPPORT" ] && {
-               # RST all tcp connections used for NLM to ensure that they do
-               # not survive in ESTABLISHED state across a failover/failback
-               # and create an ack storm
-               netstat -tn |egrep "^tcp.*\s+$ip:${LOCKD_TCPPORT}\s+.*ESTABLISHED" | awk '{print $4" "$5}' | while read dest src; do
-                       srcip=`echo $src | cut -d: -f1`
-                       srcport=`echo $src | cut -d: -f2`
-                       destip=`echo $dest | cut -d: -f1`
-                       destport=`echo $dest | cut -d: -f2`
-                       ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 
-#                      ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1
-               done
-       } > /dev/null 2>&1
-
-
-        # RST the local side for all tcp connections used for NFS to ensure 
-       # that they do not survive in ESTABLISHED state across a 
-       # failover/failback and create an ack storm
-       netstat -tn |egrep "^tcp.*\s+$ip:2049\s+.*ESTABLISHED" | awk '{print $4" "$5}' | while read dest src; do
-               srcip=`echo $src | cut -d: -f1`
-               srcport=`echo $src | cut -d: -f2`
-               destip=`echo $dest | cut -d: -f1`
-               destport=`echo $dest | cut -d: -f2`
-               ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 
-#              ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1
-       done
-
        exit 0
        ;;
 

diff --git a/ctdb/tools/ctdb_diagnostics b/ctdb/tools/ctdb_diagnostics
index 98d0bf53a624d13c5b53797ec91c80c947fe8697..7186467b7417d3b4de65d9bc38a2294ef4d49ddb 100755 (executable)
--- a/ctdb/tools/ctdb_diagnostics
+++ b/ctdb/tools/ctdb_diagnostics
@@ -123,6 +123,7 @@ show_all "/sbin/ip addr list"
 show_all "/sbin/route -n"
 show_all "crontab -l"
 show_all "sysctl -a"
+show_all "/sbin/iptables -L -n"
 
 [ -d /usr/lpp/mmfs ] && {
 cat <<EOF