From: Andrew Tridgell <tridge@samba.org>
Date: Thu, 13 Sep 2007 00:24:48 +0000 (+1000)
Subject: new approach for killing TCP connections on IP release
X-Git-Tag: tevent-0.9.20~348^2~2417
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=023b885793de737728b3b075da15d23278759794;p=thirdparty%2Fsamba.git

new approach for killing TCP connections on IP release
(This used to be ctdb commit c33a0db29b5604966f582b1f8c5fd66760c72197)
---

diff --git a/ctdb/config/events.d/10.interface b/ctdb/config/events.d/10.interface
index 76face39e99..540167deda2 100755
--- a/ctdb/config/events.d/10.interface
+++ b/ctdb/config/events.d/10.interface
@@ -20,6 +20,37 @@ shift
 	exit 0
 }
 
+################################################
+# kill off any TCP connections with the given IP
+kill_tcp_connections() {
+    _IP="$1"    
+    _failed=0
+    _killcount=0
+    netstat -tn |egrep "^tcp.*\s+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' | 
+    while read dest src; do
+	srcip=`echo $src | cut -d: -f1`
+	srcport=`echo $src | cut -d: -f2`
+	destip=`echo $dest | cut -d: -f1`
+	destport=`echo $dest | cut -d: -f2`
+	ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
+	_killcount=`expr $_killcount + 1`
+    done
+    [ _failed = 0 ] || {
+	echo "Failed to send killtcp control"
+	return;
+    }
+    _count=0
+    while netstat -tn | grep $_IP > /dev/null; do
+	sleep 1
+	_count=`expr $_count + 1`
+	[ $_count -gt 5 ] && {
+	    echo "Timed out killing tcp connections for IP $_IP"
+	    return;
+	}
+    done
+    echo "`date` killed $_killcount TCP connections to released IP $_IP"
+}
+
 case $cmd in 
      #############################
      # called when ctdbd starts up
@@ -29,11 +60,6 @@ case $cmd in
 	[ -f /proc/sys/net/ipv4/conf/all/arp_filter ] && {
 	    echo 1 > /proc/sys/net/ipv4/conf/all/arp_filter
 	}
-	# make sure we don't reply to arp requests for IPs we have moved to scope
-	# host on loopback
-	[ -f /proc/sys/net/ipv4/conf/all/arp_ignore ] && {
-	    echo 3 > /proc/sys/net/ipv4/conf/all/arp_ignore
-	}
 	;;
 
 
@@ -53,11 +79,12 @@ case $cmd in
 		 echo "`/bin/date` Failed to bringup interface $iface"
 		 exit 1
 	}
-	/sbin/ip addr del $ip/32 dev lo >/dev/null 2>/dev/null
 	/sbin/ip addr add $ip/$maskbits dev $iface || {
 		 echo "`/bin/date` Failed to add $ip/$maskbits on dev $iface"
 		 exit 1
 	}
+	# cope with the script being killed while we have the interface blocked
+	/sbin/iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
 
 	# flush our route cache
 	echo 1 > /proc/sys/net/ipv4/route/flush
@@ -71,16 +98,32 @@ case $cmd in
 	   echo "`/bin/date` must supply interface, IP and maskbits"
 	   exit 1
 	fi
+
+	# releasing an IP is a bit more complex than it seems. Once the IP
+	# is released, any open tcp connections to that IP on this host will end
+	# up being stuck. Some of them (such as NFS connections) will be unkillable
+	# so we need to use the killtcp ctdb function to kill them off. We also
+	# need to make sure that no new connections get established while we are 
+	# doing this! So what we do is this:
+	# 1) firewall this IP, so no new external packets arrive for it
+	# 2) use netstat -tn to find existing connections, and kill them 
+	# 3) remove the IP from the interface
+	# 4) remove the firewall rule
 	iface=$1
 	ip=$2
 	maskbits=$3
-	/sbin/ip addr del $ip/$maskbits dev $iface || {
+
+	failed=0
+	# we do an extra delete to cope with the script being killed
+	/sbin/iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
+	/sbin/iptables -I INPUT -i $iface -d $ip -j DROP
+	kill_tcp_connections $ip
+	/sbin/ip addr del $ip/$maskbits dev $iface || failed=1
+	/sbin/iptables -D INPUT -i $iface -d $ip -j DROP
+	[ $failed = 0 ] || {
 		 echo "`/bin/date` Failed to del $ip on dev $iface"
 		 exit 1
 	}
-	# we put the IP on loopback so our killtcp code can work, this allows
-	# us to avoid restarting the NFS server when we release an IP
-	/sbin/ip addr add $ip/32 dev lo scope host >/dev/null 2>/dev/null
 
 	# flush our route cache
 	echo 1 > /proc/sys/net/ipv4/route/flush
diff --git a/ctdb/config/events.d/60.nfs b/ctdb/config/events.d/60.nfs
index 8c9e7ebd780..f19dfea1494 100755
--- a/ctdb/config/events.d/60.nfs
+++ b/ctdb/config/events.d/60.nfs
@@ -55,37 +55,7 @@ case $cmd in
 	maskbits=$3
 
 	echo $ip >> /etc/ctdb/state/statd/restart
-
 	/bin/rm -f /etc/ctdb/state/statd/ip/$ip
-
-	# RST all tcp connections to the lockmanager
-	[ ! -z "$LOCKD_TCPPORT" ] && {
-	        # RST all tcp connections used for NLM to ensure that they do
-		# not survive in ESTABLISHED state across a failover/failback
-		# and create an ack storm
-		netstat -tn |egrep "^tcp.*\s+$ip:${LOCKD_TCPPORT}\s+.*ESTABLISHED" | awk '{print $4" "$5}' | while read dest src; do
-			srcip=`echo $src | cut -d: -f1`
-			srcport=`echo $src | cut -d: -f2`
-			destip=`echo $dest | cut -d: -f1`
-			destport=`echo $dest | cut -d: -f2`
-			ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 
-#			ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1
-		done
-	} > /dev/null 2>&1
-
-
-        # RST the local side for all tcp connections used for NFS to ensure 
-	# that they do not survive in ESTABLISHED state across a 
-	# failover/failback and create an ack storm
-	netstat -tn |egrep "^tcp.*\s+$ip:2049\s+.*ESTABLISHED" | awk '{print $4" "$5}' | while read dest src; do
-		srcip=`echo $src | cut -d: -f1`
-		srcport=`echo $src | cut -d: -f2`
-		destip=`echo $dest | cut -d: -f1`
-		destport=`echo $dest | cut -d: -f2`
-		ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 
-#		ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1
-	done
-
 	exit 0
 	;;
 
diff --git a/ctdb/tools/ctdb_diagnostics b/ctdb/tools/ctdb_diagnostics
index 98d0bf53a62..7186467b741 100755
--- a/ctdb/tools/ctdb_diagnostics
+++ b/ctdb/tools/ctdb_diagnostics
@@ -123,6 +123,7 @@ show_all "/sbin/ip addr list"
 show_all "/sbin/route -n"
 show_all "crontab -l"
 show_all "sysctl -a"
+show_all "/sbin/iptables -L -n"
 
 [ -d /usr/lpp/mmfs ] && {
 cat <<EOF