Fix oinkmaster patch.

[ipfire-2.x.git] / src / patches / suse-2.6.27.39 / patches.drivers / cxgb3i-fix-skb-overrun

Subject: cxgb3i - fixes over-run of skb MAX_SKB_FRAGS
From: Karen Xie <kxie@chelsio.com>
References: bnc#468314

This patch fixes the over-run of skb's MAX_SKB_FRAGS between the cxgb3i and
cxgb3 driver on PPC64 systems.

Signed-off-by: Karen Xie <kxie@chelsio.com>
Signed-off-by: Chandra Seetharaman <sekharan@us.ibm.com>
Acked-by: Hannes Reinecke <hare@suse.de>
---

---
 drivers/scsi/cxgb3i/cxgb3i.h         |  148 ++++++++++++++
 drivers/scsi/cxgb3i/cxgb3i_init.c    |    4 
 drivers/scsi/cxgb3i/cxgb3i_iscsi.c   |  138 ++++++-------
 drivers/scsi/cxgb3i/cxgb3i_offload.c |  126 +++++++++---
 drivers/scsi/cxgb3i/cxgb3i_offload.h |    6 
 drivers/scsi/cxgb3i/cxgb3i_ulp2.c    |  353 ++++++++++++++++++++++++-----------
 drivers/scsi/cxgb3i/cxgb3i_ulp2.h    |    4 
 7 files changed, 553 insertions(+), 226 deletions(-)

--- a/drivers/scsi/cxgb3i/cxgb3i.h
+++ b/drivers/scsi/cxgb3i/cxgb3i.h
@@ -36,6 +36,12 @@
 #define CXGB3I_MAX_TARGET	CXGB3I_MAX_CONN
 #define CXGB3I_MAX_LUN		512
 #define ISCSI_PDU_HEADER_MAX	(56 + 256) /* bhs + digests + ahs */
+#define ULP2_MAX_PKT_SIZE		16224
+#define ISCSI_PDU_NONPAYLOAD_MAX  \
+	(sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE + 2*ISCSI_DIGEST_SIZE)
+#define ULP2_MAX_PDU_PAYLOAD \
+	(ULP2_MAX_PKT_SIZE - ISCSI_PDU_NONPAYLOAD_MAX)
+
 
 struct cxgb3i_adapter;
 struct cxgb3i_hba;
@@ -53,12 +59,11 @@ struct cxgb3i_endpoint;
  *
  */
 struct cxgb3i_tag_format {
-	unsigned char idx_bits;
-	unsigned char age_bits;
+	unsigned char sw_bits;
 	unsigned char rsvd_bits;
 	unsigned char rsvd_shift;
+	unsigned char filler[1];
 	u32 rsvd_mask;
-	u32 rsvd_tag_mask;
 };
 
 /**
@@ -95,11 +100,137 @@ struct cxgb3i_ddp_info {
 	unsigned int ulimit;
 	unsigned int nppods;
 	unsigned int idx_last;
+	unsigned char idx_bits;
+	unsigned char filler[3];
+	u32 idx_mask;
+	u32 rsvd_tag_mask;
 	spinlock_t map_lock;
 	struct cxgb3i_gather_list **gl_map;
 	struct sk_buff **gl_skb;
 };
 
+/*
+ * cxgb3i ddp tag are 32 bits, it consists of reserved bits used by h/w and
+ * non-reserved bits that can be used by the iscsi s/w.
+ * The reserved bits are identified by the rsvd_bits and rsvd_shift fields
+ * in struct cxgb3i_tag_format.
+ *
+ * The upper most reserved bit can be used to check if a tag is ddp tag or not:
+ * 	if the bit is 0, the tag is a valid ddp tag
+ */
+
+/**
+ * cxgb3i_is_ddp_tag - check if a given tag is a hw/ddp tag
+ * @tformat: tag format information
+ * @tag: tag to be checked
+ *
+ * return true if the tag is a ddp tag, false otherwise.
+ */
+static inline int cxgb3i_is_ddp_tag(struct cxgb3i_tag_format *tformat, u32 tag)
+{
+	return !(tag & (1 << (tformat->rsvd_bits + tformat->rsvd_shift - 1)));
+}
+
+/**
+ * cxgb3i_sw_tag_usable - check if a given s/w tag has enough bits left for
+ *			  the reserved/hw bits
+ * @tformat: tag format information
+ * @sw_tag: s/w tag to be checked
+ *
+ * return true if the tag is a ddp tag, false otherwise.
+ */
+static inline int cxgb3i_sw_tag_usable(struct cxgb3i_tag_format *tformat,
+					u32 sw_tag)
+{
+	sw_tag >>= (32 - tformat->rsvd_bits);
+	return !sw_tag;
+}
+
+/**
+ * cxgb3i_set_non_ddp_tag - mark a given s/w tag as an invalid ddp tag
+ * @tformat: tag format information
+ * @sw_tag: s/w tag to be checked
+ *
+ * insert 1 at the upper most reserved bit to mark it as an invalid ddp tag.
+ */
+static inline u32 cxgb3i_set_non_ddp_tag(struct cxgb3i_tag_format *tformat,
+					 u32 sw_tag)
+{
+	unsigned char shift = tformat->rsvd_bits + tformat->rsvd_shift - 1;
+	u32 mask = (1 << shift) - 1;
+
+	if (sw_tag && (sw_tag & ~mask)) {
+		u32 v1 = sw_tag & ((1 << shift) - 1);
+		u32 v2 = (sw_tag >> (shift - 1)) << shift;
+
+		return v2 | v1 | 1 << shift;
+	}
+	return sw_tag | 1 << shift;
+}
+
+/**
+ * cxgb3i_ddp_tag_base - shift the s/w tag bits so that reserved bits are not
+ *			 used.
+ * @tformat: tag format information
+ * @sw_tag: s/w tag to be checked
+ */
+static inline u32 cxgb3i_ddp_tag_base(struct cxgb3i_tag_format *tformat,
+				      u32 sw_tag)
+{
+	u32 mask = (1 << tformat->rsvd_shift) - 1;
+
+	if (sw_tag && (sw_tag & ~mask)) {
+		u32 v1 = sw_tag & mask;
+		u32 v2 = sw_tag >> tformat->rsvd_shift;
+
+		v2 <<= tformat->rsvd_shift + tformat->rsvd_bits;
+		return v2 | v1;
+	}
+	return sw_tag;
+}
+
+/**
+ * cxgb3i_tag_rsvd_bits - get the reserved bits used by the h/w
+ * @tformat: tag format information
+ * @tag: tag to be checked
+ *
+ * return the reserved bits in the tag
+ */
+static inline u32 cxgb3i_tag_rsvd_bits(struct cxgb3i_tag_format *tformat,
+				       u32 tag)
+{
+	if (cxgb3i_is_ddp_tag(tformat, tag))
+		return (tag >> tformat->rsvd_shift) & tformat->rsvd_mask;
+	return 0;
+}
+
+/**
+ * cxgb3i_tag_nonrsvd_bits - get the non-reserved bits used by the s/w
+ * @tformat: tag format information
+ * @tag: tag to be checked
+ *
+ * return the non-reserved bits in the tag.
+ */
+static inline u32 cxgb3i_tag_nonrsvd_bits(struct cxgb3i_tag_format *tformat,
+					  u32 tag)
+{
+	unsigned char shift = tformat->rsvd_bits + tformat->rsvd_shift - 1;
+	u32 v1, v2;
+
+	if (cxgb3i_is_ddp_tag(tformat, tag)) {
+		v1 = tag & ((1 << tformat->rsvd_shift) - 1);
+		v2 = (tag >> (shift + 1)) << tformat->rsvd_shift;
+	} else {
+		u32 mask = (1 << shift) - 1;
+
+		tag &= ~(1 << shift);
+		v1 = tag & mask;
+		v2 = (tag >> 1) & ~mask;
+	}
+	return v1 | v2;
+}
+
+
 /**
  * struct cxgb3i_hba - cxgb3i iscsi structure (per port)
  *
@@ -146,16 +277,22 @@ struct cxgb3i_adapter {
  * struct cxgb3i_conn - cxgb3i iscsi connection
  *
  * @tcp_conn:	pointer to iscsi_tcp_conn structure
- * @listhead:	list head to link elements
+ * @list_head:	list head to link elements
+ * @cep:	pointer to iscsi_endpoint structure
  * @conn:	pointer to iscsi_conn structure
  * @hba:	pointer to the hba this conn. is going through
+ * @task_idx_bits: # of bits needed for session->cmds_max
+ * @frags:	temp. holding area for tx coalesced sg list pages.
  */
+#define TX_PDU_PAGES_MAX   (16384/512 + 1)
 struct cxgb3i_conn {
 	struct iscsi_tcp_conn tcp_conn;
 	struct list_head list_head;
 	struct cxgb3i_endpoint *cep;
 	struct iscsi_conn *conn;
 	struct cxgb3i_hba *hba;
+	unsigned int task_idx_bits;
+	skb_frag_t frags[TX_PDU_PAGES_MAX];
 };
 
 /**
@@ -190,8 +327,7 @@ void cxgb3i_hba_host_remove(struct cxgb3
 int cxgb3i_ulp2_init(void);
 void cxgb3i_ulp2_cleanup(void);
 int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *, int, int);
-void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32,
-			    struct scatterlist *, unsigned int);
+void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32);
 u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *, unsigned int,
 			   u32, unsigned int, struct scatterlist *,
 			   unsigned int, int);
--- a/drivers/scsi/cxgb3i/cxgb3i_init.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_init.c
@@ -11,8 +11,8 @@
 
 #include "cxgb3i.h"
 
-#define DRV_MODULE_NAME         "cxgb3i"
-#define DRV_MODULE_VERSION	"0.1.0"
+#define DRV_MODULE_NAME		"cxgb3i"
+#define DRV_MODULE_VERSION	"0.9.0"
 #define DRV_MODULE_RELDATE	"Jun. 1, 2008"
 
 static char version[] =
--- a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
@@ -36,10 +36,10 @@
 #define cxgb3i_api_debug(fmt...)
 #endif
 
-#define align_to_4k_boundary(n)	\
-	do { \
-		n = (n) & ~((1 << 12) - 1); \
-	} while(0)
+/*
+ * align pdu size to multiple of 512 for better performance
+ */
+#define align_pdu_size(n) do { n = (n) & (~511); } while (0)
 
 static struct scsi_transport_template *cxgb3i_scsi_transport;
 static struct scsi_host_template cxgb3i_host_template;
@@ -102,7 +102,7 @@ void cxgb3i_adapter_remove(struct t3cdev
 	struct cxgb3i_adapter *snic;
 
 	/* remove from the list */
-	read_lock(&cxgb3i_snic_rwlock);
+	write_lock(&cxgb3i_snic_rwlock);
 	list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
 		if (snic->tdev == t3dev) {
 			list_del(&snic->list_head);
@@ -295,6 +295,8 @@ static void cxgb3i_ep_disconnect(struct
 		 * stop the xmit path so the xmit_segment function is
 		 * not being called
 		 */
+		iscsi_suspend_tx(cconn->conn);
+
 		write_lock_bh(&cep->c3cn->callback_lock);
 		set_bit(ISCSI_SUSPEND_BIT, &cconn->conn->suspend_rx);
 		cep->c3cn->user_data = NULL;
@@ -391,20 +393,17 @@ static void cxgb3i_session_destroy(struc
 static inline int cxgb3i_conn_max_xmit_dlength(struct iscsi_conn *conn)
 {
 	struct cxgb3i_conn *cconn = conn->dd_data;
-	unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_SIZE,
-					   cconn->hba->snic->tx_max_size -
-					   ISCSI_PDU_HEADER_MAX);
-
-	cxgb3i_api_debug("conn 0x%p, max xmit %u.\n",
-			 conn, conn->max_xmit_dlength);
+	unsigned int max = min_t(unsigned int,
+				ULP2_MAX_PDU_PAYLOAD,
+				cconn->hba->snic->tx_max_size -
+					ISCSI_PDU_NONPAYLOAD_MAX);
 
 	if (conn->max_xmit_dlength)
 		conn->max_xmit_dlength = min_t(unsigned int,
-						conn->max_xmit_dlength, max);
+					conn->max_xmit_dlength, max);
 	else
 		conn->max_xmit_dlength = max;
-
-	align_to_4k_boundary(conn->max_xmit_dlength);
+	align_pdu_size(conn->max_xmit_dlength);
 
 	cxgb3i_api_debug("conn 0x%p, set max xmit %u.\n",
 			 conn, conn->max_xmit_dlength);
@@ -415,14 +414,10 @@ static inline int cxgb3i_conn_max_xmit_d
 static inline int cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
 {
 	struct cxgb3i_conn *cconn = conn->dd_data;
-	unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_SIZE,
-					   cconn->hba->snic->rx_max_size -
-					   ISCSI_PDU_HEADER_MAX);
-
-	cxgb3i_api_debug("conn 0x%p, max recv %u.\n",
-			 conn, conn->max_recv_dlength);
-
-	align_to_4k_boundary(max);
+	unsigned int max = min_t(unsigned int,
+				ULP2_MAX_PDU_PAYLOAD,
+				cconn->hba->snic->tx_max_size -
+					ISCSI_PDU_NONPAYLOAD_MAX);
 
 	if (conn->max_recv_dlength) {
 		if (conn->max_recv_dlength > max) {
@@ -433,9 +428,9 @@ static inline int cxgb3i_conn_max_recv_d
 		}
 		conn->max_recv_dlength = min_t(unsigned int,
 						conn->max_recv_dlength, max);
-		align_to_4k_boundary(conn->max_recv_dlength);
 	} else
 		conn->max_recv_dlength = max;
+	align_pdu_size(conn->max_recv_dlength);
 
 	cxgb3i_api_debug("conn 0x%p, set max recv %u.\n",
 			 conn, conn->max_recv_dlength);
@@ -516,12 +511,14 @@ static int cxgb3i_conn_bind(struct iscsi
 
 	cep = ep->dd_data;
 	c3cn = cep->c3cn;
+	/* calculate the tag idx bits needed for this conn based on cmds_max */
+	cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1;
 
-	read_lock(&c3cn->callback_lock);
+	write_lock(&c3cn->callback_lock);
 	/* mnc: TODO don't abuse iscsi_tcp fields */
 	tcp_conn->sock = (struct socket *)c3cn;
 	c3cn->user_data = conn;
-	read_unlock(&c3cn->callback_lock);
+	write_unlock(&c3cn->callback_lock);
 
 	cconn->hba = cep->hba;
 	cconn->cep = cep;
@@ -609,11 +606,13 @@ static int cxgb3i_conn_set_param(struct
 			return -ENOMEM;
 	case ISCSI_PARAM_MAX_RECV_DLENGTH:
 		err = iscsi_set_param(cls_conn, param, buf, buflen);
-		err = cxgb3i_conn_max_recv_dlength(conn);
+		if (!err)
+			err = cxgb3i_conn_max_recv_dlength(conn);
 		break;
 	case ISCSI_PARAM_MAX_XMIT_DLENGTH:
 		err = iscsi_set_param(cls_conn, param, buf, buflen);
-		err = cxgb3i_conn_max_xmit_dlength(conn);
+		if (!err)
+			err = cxgb3i_conn_max_xmit_dlength(conn);
 		break;
 	default:
 		return iscsi_set_param(cls_conn, param, buf, buflen);
@@ -718,49 +717,23 @@ static void cxgb3i_conn_get_stats(struct
 	stats->custom[0].value = conn->eh_abort_cnt;
 }
 
-static inline u32 tag_base(struct cxgb3i_tag_format *format,
-			   unsigned int idx, unsigned int age)
-{
-	u32 sw_bits = idx | (age << format->idx_bits);
-	u32 tag = sw_bits >> format->rsvd_shift;
-
-	tag <<= format->rsvd_bits + format->rsvd_shift;
-	tag |= sw_bits & ((1 << format->rsvd_shift) - 1);
-	return tag;
-}
-
-static inline void cxgb3i_parse_tag(struct cxgb3i_tag_format *format,
-				    u32 tag, u32 *rsvd_bits, u32 *sw_bits)
-{
-	if (rsvd_bits)
-		*rsvd_bits = (tag >> format->rsvd_shift) & format->rsvd_mask;
-	if (sw_bits) {
-		*sw_bits = (tag >> (format->rsvd_shift + format->rsvd_bits))
-			    << format->rsvd_shift;
-		*sw_bits |= tag & ((1 << format->rsvd_shift) - 1);
-	}
-
-	cxgb3i_tag_debug("parse tag 0x%x, rsvd 0x%x, sw  0x%x.\n",
-			 tag, rsvd_bits ? *rsvd_bits : 0xFFFFFFFF,
-			 sw_bits ? *sw_bits : 0xFFFFFFFF);
-}
-
-
 static void cxgb3i_parse_itt(struct iscsi_conn *conn, itt_t itt,
 			     int *idx, int *age)
 {
 	struct cxgb3i_conn *cconn = conn->dd_data;
 	struct cxgb3i_adapter *snic = cconn->hba->snic;
+	u32 tag = itt;
 	u32 sw_bits;
 
-	cxgb3i_parse_tag(&snic->tag_format, itt, NULL, &sw_bits);
+	sw_bits = cxgb3i_tag_nonrsvd_bits(&snic->tag_format, tag);
 	if (idx)
-		*idx = sw_bits & ISCSI_ITT_MASK;
+		*idx = sw_bits & ((1 << cconn->task_idx_bits) - 1);
 	if (age)
-		*age = (sw_bits >> snic->tag_format.idx_bits) & ISCSI_AGE_MASK;
+		*age = (sw_bits >> cconn->task_idx_bits) & ISCSI_AGE_MASK;
 
-	cxgb3i_tag_debug("parse itt 0x%x, idx 0x%x, age 0x%x.\n",
-			 itt,  idx ? *idx : 0xFFFFF, age ? *age : 0xFF);
+	cxgb3i_tag_debug("parse tag 0x%x/0x%x, sw 0x%x, idx 0x%x, age 0x%x.\n",
+			tag, itt, sw_bits, idx ? *idx : 0xFFFFF,
+			age ? *age : 0xFF);
 }
 
 static int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
@@ -771,26 +744,40 @@ static int cxgb3i_reserve_itt(struct isc
 	struct cxgb3i_conn *cconn = conn->dd_data;
 	struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
 	struct cxgb3i_adapter *snic = cconn->hba->snic;
-	u32 sw_tag = tag_base(&snic->tag_format, task->itt, sess->age);
+	struct cxgb3i_tag_format *tformat = &snic->tag_format;
+	u32 sw_tag = (sess->age << cconn->task_idx_bits) | task->itt;
 	u32 tag = RESERVED_ITT;
 
-	if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE)) {
+	if (sc && (scsi_bidi_cmnd(sc) ||
+		sc->sc_data_direction == DMA_FROM_DEVICE) &&
+		cxgb3i_sw_tag_usable(tformat, sw_tag)) {
 		struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
-		tag =
-		    cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag,
+		tag = cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag,
 					   scsi_in(sc)->length,
 					   scsi_in(sc)->table.sgl,
 					   scsi_in(sc)->table.nents,
 					   GFP_ATOMIC);
 	}
 	if (tag == RESERVED_ITT)
-		tag = sw_tag | (snic->tag_format.rsvd_mask <<
-				snic->tag_format.rsvd_shift);
+		tag = cxgb3i_set_non_ddp_tag(tformat, sw_tag);
+	/* the itt need to sent in big-endian order */
 	*hdr_itt = htonl(tag);
 
-	cxgb3i_tag_debug("new tag 0x%x/0x%x (itt 0x%x, age 0x%x).\n",
-			 tag, *hdr_itt, task->itt, sess->age);
-
+	if (sc) {
+		if (sc->sc_data_direction == DMA_FROM_DEVICE)
+			cxgb3i_tag_debug("read, len %u, tag 0x%x/0x%x "
+					 "(itt 0x%x, age 0x%x, sw 0x%x).\n",
+					 scsi_in(sc)->length, tag, *hdr_itt,
+					 task->itt, sess->age, sw_tag);
+		else
+			cxgb3i_tag_debug("write, len %u, tag 0x%x/0x%x "
+					 "(itt 0x%x, age 0x%x, sw 0x%x).\n",
+					 scsi_out(sc)->length, tag, *hdr_itt,
+					 task->itt, sess->age, sw_tag);
+	} else
+		cxgb3i_tag_debug("ctrl, tag 0x%x/0x%x (itt 0x%x, age 0x%x, "
+				 "sw 0x%x).\n",
+				 tag, *hdr_itt, task->itt, sess->age, sw_tag);
 	return 0;
 }
 
@@ -800,14 +787,15 @@ static void cxgb3i_release_itt(struct is
 	struct iscsi_conn *conn = task->conn;
 	struct cxgb3i_conn *cconn = conn->dd_data;
 	struct cxgb3i_adapter *snic = cconn->hba->snic;
+	struct cxgb3i_tag_format *tformat = &snic->tag_format;
 	u32 tag = ntohl(hdr_itt);
 
-	cxgb3i_tag_debug("release tag 0x%x.\n", tag);
+	cxgb3i_tag_debug("release %s tag 0x%x.\n", sc ? "scsi" : "ctrl", tag);
 
-	if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE))
-		cxgb3i_ddp_tag_release(snic, tag,
-				       scsi_in(sc)->table.sgl,
-				       scsi_in(sc)->table.nents);
+	if (sc && (scsi_bidi_cmnd(sc) ||
+		sc->sc_data_direction == DMA_FROM_DEVICE) &&
+		cxgb3i_is_ddp_tag(tformat, tag))
+		cxgb3i_ddp_tag_release(snic, tag);
 }
 
 /**
@@ -820,7 +808,7 @@ static struct scsi_host_template cxgb3i_
 	.proc_name = "cxgb3i",
 	.queuecommand = iscsi_queuecommand,
 	.change_queue_depth = iscsi_change_queue_depth,
-	.can_queue = 128 * (ISCSI_DEF_XMIT_CMDS_MAX - 1),
+	.can_queue = CXGB3I_SCSI_QDEPTH_DFLT - 1,
 	.sg_tablesize = SG_ALL,
 	.max_sectors = 0xFFFF,
 	.cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
--- a/drivers/scsi/cxgb3i/cxgb3i_offload.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
@@ -22,19 +22,19 @@
 #include "cxgb3i_ulp2.h"
 
 #ifdef __DEBUG_C3CN_CONN__
-#define c3cn_conn_debug         cxgb3i_log_debug
+#define c3cn_conn_debug		cxgb3i_log_debug
 #else
 #define c3cn_conn_debug(fmt...)
 #endif
 
 #ifdef __DEBUG_C3CN_TX__
-#define c3cn_tx_debug         cxgb3i_log_debug
+#define c3cn_tx_debug		cxgb3i_log_debug
 #else
 #define c3cn_tx_debug(fmt...)
 #endif
 
 #ifdef __DEBUG_C3CN_RX__
-#define c3cn_rx_debug         cxgb3i_log_debug
+#define c3cn_rx_debug		cxgb3i_log_debug
 #else
 #define c3cn_rx_debug(fmt...)
 #endif
@@ -42,9 +42,9 @@
 /*
  * module parameters releated to offloaded iscsi connection
  */
-static int cxgb3_rcv_win = 256 * 1024;
+static int cxgb3_rcv_win = 128 * 1024;
 module_param(cxgb3_rcv_win, int, 0644);
-MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
+MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=128KB)");
 
 static int cxgb3_snd_win = 64 * 1024;
 module_param(cxgb3_snd_win, int, 0644);
@@ -456,12 +456,9 @@ static unsigned int wrlen __read_mostly;
  * The number of WRs needed for an skb depends on the number of fragments
  * in the skb and whether it has any payload in its main body.  This maps the
  * length of the gather list represented by an skb into the # of necessary WRs.
- *
- * The max. length of an skb is controlled by the max pdu size which is ~16K.
- * Also, assume the min. fragment length is the sector size (512), then add
- * extra fragment counts for iscsi bhs and payload padding.
+ * The extra two fragments are for iscsi bhs and payload padding.
  */
-#define SKB_WR_LIST_SIZE	(16384/512 + 3)
+#define SKB_WR_LIST_SIZE	(MAX_SKB_FRAGS + 2)
 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
 
 static void s3_init_wr_tab(unsigned int wr_len)
@@ -484,7 +481,7 @@ static void s3_init_wr_tab(unsigned int
 
 static inline void reset_wr_list(struct s3_conn *c3cn)
 {
-	c3cn->wr_pending_head = NULL;
+	c3cn->wr_pending_head = c3cn->wr_pending_tail = NULL;
 }
 
 /*
@@ -495,7 +492,7 @@ static inline void reset_wr_list(struct
 static inline void enqueue_wr(struct s3_conn *c3cn,
 			      struct sk_buff *skb)
 {
-	skb->sp = NULL;
+	skb_wr_next(skb) = NULL;
 
 	/*
 	 * We want to take an extra reference since both us and the driver
@@ -508,10 +505,22 @@ static inline void enqueue_wr(struct s3_
 	if (!c3cn->wr_pending_head)
 		c3cn->wr_pending_head = skb;
 	else
-		c3cn->wr_pending_tail->sp = (void *)skb;
+		skb_wr_next(c3cn->wr_pending_tail) = skb;
 	c3cn->wr_pending_tail = skb;
 }
 
+static int count_pending_wrs(struct s3_conn *c3cn)
+{
+	int n = 0;
+	const struct sk_buff *skb = c3cn->wr_pending_head;
+
+	while (skb) {
+		n += skb->csum;
+		skb = skb_wr_next(skb);
+	}
+	return n;
+}
+
 static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
 {
 	return c3cn->wr_pending_head;
@@ -528,8 +537,8 @@ static inline struct sk_buff *dequeue_wr
 
 	if (likely(skb)) {
 		/* Don't bother clearing the tail */
-		c3cn->wr_pending_head = (struct sk_buff *)skb->sp;
-		skb->sp = NULL;
+		c3cn->wr_pending_head = skb_wr_next(skb);
+		skb_wr_next(skb) = NULL;
 	}
 	return skb;
 }
@@ -542,13 +551,15 @@ static void purge_wr_queue(struct s3_con
 }
 
 static inline void make_tx_data_wr(struct s3_conn *c3cn, struct sk_buff *skb,
-				   int len)
+					int len, int req_completion)
 {
 	struct tx_data_wr *req;
 
 	skb_reset_transport_header(skb);
 	req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
-	req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
+
+	req->wr_hi = htonl((V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)) |
+				(req_completion ? F_WR_COMPL : 0));
 	req->wr_lo = htonl(V_WR_TID(c3cn->tid));
 	req->sndseq = htonl(c3cn->snd_nxt);
 	/* len includes the length of any HW ULP additions */
@@ -556,11 +567,11 @@ static inline void make_tx_data_wr(struc
 	req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx));
 	/* V_TX_ULP_SUBMODE sets both the mode and submode */
 	req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) |
-			   V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
+			V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
 
 	if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
-		req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
-				    V_TX_CPU_IDX(c3cn->qset));
+			req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
+					V_TX_CPU_IDX(c3cn->qset));
 		/* Sendbuffer is in units of 32KB. */
 		req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
 		c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
@@ -591,7 +602,7 @@ static int c3cn_push_tx_frames(struct s3
 
 	if (unlikely(c3cn->state == C3CN_STATE_CONNECTING ||
 		     c3cn->state == C3CN_STATE_CLOSE_WAIT_1 ||
-		     c3cn->state == C3CN_STATE_ABORTING)) {
+		     c3cn->state >= C3CN_STATE_ABORTING)) {
 		c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
 			      c3cn, c3cn->state);
 		return 0;
@@ -626,19 +637,22 @@ static int c3cn_push_tx_frames(struct s3
 		c3cn->wr_unacked += wrs_needed;
 		enqueue_wr(c3cn, skb);
 
-		if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
-			len += ulp_extra_len(skb);
-			make_tx_data_wr(c3cn, skb, len);
-			c3cn->snd_nxt += len;
-			if ((req_completion
-			     && c3cn->wr_unacked == wrs_needed)
-			    || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL)
-			    || c3cn->wr_unacked >= c3cn->wr_max / 2) {
-				struct work_request_hdr *wr = cplhdr(skb);
+		c3cn_tx_debug("c3cn 0x%p, enqueue, skb len %u/%u, frag %u, "
+				"wr %d, left %u, unack %u.\n",
+				c3cn, skb->len, skb->data_len, frags,
+				wrs_needed, c3cn->wr_avail, c3cn->wr_unacked);
 
-				wr->wr_hi |= htonl(F_WR_COMPL);
+		if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
+			if ((req_completion &&
+				c3cn->wr_unacked == wrs_needed) ||
+				(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL) ||
+				c3cn->wr_unacked >= c3cn->wr_max / 2) {
+				req_completion = 1;
 				c3cn->wr_unacked = 0;
 			}
+			len += ulp_extra_len(skb);
+			make_tx_data_wr(c3cn, skb, len, req_completion);
+			c3cn->snd_nxt += len;
 			CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR;
 		}
 
@@ -1153,12 +1167,28 @@ static int do_iscsi_hdr(struct t3cdev *t
  * Process an acknowledgment of WR completion.  Advance snd_una and send the
  * next batch of work requests from the write queue.
  */
+
+static void check_wr_invariants(struct s3_conn *c3cn)
+{
+	int pending = count_pending_wrs(c3cn);
+
+	if (unlikely(c3cn->wr_avail + pending != c3cn->wr_max))
+			cxgb3i_log_error("TID %u: credit imbalance: avail %u, "
+					"pending %u, total should be %u\n",
+					c3cn->tid, c3cn->wr_avail, pending,
+					c3cn->wr_max);
+}
+
 static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
 {
 	struct cpl_wr_ack *hdr = cplhdr(skb);
 	unsigned int credits = ntohs(hdr->credits);
 	u32 snd_una = ntohl(hdr->snd_una);
 
+	c3cn_tx_debug("%u WR credits, avail %u, unack %u, TID %u, state %u.\n",
+			credits, c3cn->wr_avail, c3cn->wr_unacked,
+			c3cn->tid, c3cn->state);
+
 	c3cn->wr_avail += credits;
 	if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
 		c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
@@ -1173,6 +1203,17 @@ static void process_wr_ack(struct s3_con
 			break;
 		}
 		if (unlikely(credits < p->csum)) {
+			struct tx_data_wr *w = cplhdr(p);
+			cxgb3i_log_error("TID %u got %u WR credits need %u, "
+					"len %u, main body %u, frags %u, "
+					"seq # %u, ACK una %u, ACK nxt %u, "
+					"WR_AVAIL %u, WRs pending %u\n",
+					c3cn->tid, credits, p->csum, p->len,
+					p->len - p->data_len,
+					skb_shinfo(p)->nr_frags,
+					ntohl(w->sndseq), snd_una,
+					ntohl(hdr->snd_nxt), c3cn->wr_avail,
+					count_pending_wrs(c3cn) - credits);
 			p->csum -= credits;
 			break;
 		} else {
@@ -1182,8 +1223,14 @@ static void process_wr_ack(struct s3_con
 		}
 	}
 
-	if (unlikely(before(snd_una, c3cn->snd_una)))
+	check_wr_invariants(c3cn);
+
+	if (unlikely(before(snd_una, c3cn->snd_una))) {
+		cxgb3i_log_error("TID %u, unexpected sequence # %u in WR_ACK "
+				"snd_una %u\n",
+				c3cn->tid, snd_una, c3cn->snd_una);
 		goto out_free;
+	}
 
 	if (c3cn->snd_una != snd_una) {
 		c3cn->snd_una = snd_una;
@@ -1454,11 +1501,14 @@ static void init_offload_conn(struct s3_
 			      struct dst_entry *dst)
 {
 	BUG_ON(c3cn->cdev != cdev);
-	c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs;
+	c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs - 1;
 	c3cn->wr_unacked = 0;
 	c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
 
 	reset_wr_list(c3cn);
+
+	c3cn_conn_debug("c3cn 0x%p, wr max %u, avail %u.\n",
+					c3cn, c3cn->wr_max, c3cn->wr_avail);
 }
 
 static int initiate_act_open(struct s3_conn *c3cn, struct net_device *dev)
@@ -1673,9 +1723,17 @@ int cxgb3i_c3cn_send_pdus(struct s3_conn
 		goto out_err;
 	}
 
-	err = -EPIPE;
 	if (c3cn->err) {
 		c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn, c3cn->err);
+		err = -EPIPE;
+		goto out_err;
+	}
+
+	if (c3cn->write_seq - c3cn->snd_una >= cxgb3_snd_win) {
+		c3cn_tx_debug("c3cn 0x%p, snd %u - %u > %u.\n",
+				c3cn, c3cn->write_seq, c3cn->snd_una,
+				cxgb3_snd_win);
+		err = -EAGAIN;
 		goto out_err;
 	}
 
--- a/drivers/scsi/cxgb3i/cxgb3i_offload.h
+++ b/drivers/scsi/cxgb3i/cxgb3i_offload.h
@@ -180,7 +180,7 @@ void cxgb3i_c3cn_release(struct s3_conn
  * @seq:	tcp sequence number
  * @ddigest:	pdu data digest
  * @pdulen:	recovered pdu length
- * @ulp_data:	scratch area for ULP
+ * @wr_next:	scratch area for tx wr
  */
 struct cxgb3_skb_cb {
 	__u8 flags;
@@ -188,7 +188,7 @@ struct cxgb3_skb_cb {
 	__u32 seq;
 	__u32 ddigest;
 	__u32 pdulen;
-	__u8 ulp_data[16];
+	struct sk_buff *wr_next;
 };
 
 #define CXGB3_SKB_CB(skb)	((struct cxgb3_skb_cb *)&((skb)->cb[0]))
@@ -196,7 +196,7 @@ struct cxgb3_skb_cb {
 #define skb_ulp_mode(skb)	(CXGB3_SKB_CB(skb)->ulp_mode)
 #define skb_ulp_ddigest(skb)	(CXGB3_SKB_CB(skb)->ddigest)
 #define skb_ulp_pdulen(skb)	(CXGB3_SKB_CB(skb)->pdulen)
-#define skb_ulp_data(skb)	(CXGB3_SKB_CB(skb)->ulp_data)
+#define skb_wr_next(skb)	(CXGB3_SKB_CB(skb)->wr_next)
 
 enum c3cb_flags {
 	C3CB_FLAG_NEED_HDR = 1 << 0,	/* packet needs a TX_DATA_WR header */
--- a/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
@@ -51,6 +51,7 @@ static unsigned char ddp_page_shift[ULP2
 static unsigned char sw_tag_idx_bits;
 static unsigned char sw_tag_age_bits;
 static unsigned char page_idx = ULP2_PGIDX_MAX;
+static unsigned int skb_copymax = SKB_MAX_HEAD(TX_HEADER_LEN);
 
 static void cxgb3i_ddp_page_init(void)
 {
@@ -59,6 +60,10 @@ static void cxgb3i_ddp_page_init(void)
 	sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
 	sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
 
+	cxgb3i_log_info("tag itt 0x%x, %u bits, age 0x%x, %u bits.\n",
+					ISCSI_ITT_MASK, sw_tag_idx_bits,
+					ISCSI_AGE_MASK, sw_tag_age_bits);
+
 	for (i = 0; i < ULP2_PGIDX_MAX; i++) {
 		if (PAGE_SIZE == (1UL << ddp_page_shift[i])) {
 			page_idx = i;
@@ -312,7 +317,6 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i
 				 page_idx, sgcnt, xferlen, ULP2_DDP_THRESHOLD);
 		return RESERVED_ITT;
 	}
-	return RESERVED_ITT;
 
 	gl = ddp_make_gl(xferlen, sgl, sgcnt, gfp);
 	if (!gl) {
@@ -322,9 +326,9 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i
 	}
 
 	npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
-	idx_max = ddp->nppods - npods + 1;
+	idx_max = ddp->nppods - npods;
 
-	if (ddp->idx_last == ddp->nppods)
+	if (ddp->idx_last >= idx_max)
 		idx = ddp_find_unused_entries(ddp, 0, idx_max, npods, gl);
 	else {
 		idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1, idx_max,
@@ -346,11 +350,12 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i
 	if (ddp_gl_map(snic->pdev, gl) < 0)
 		goto unmap_sgl;
 
-	tag = sw_tag | (idx << snic->tag_format.rsvd_shift);
+	tag = cxgb3i_ddp_tag_base(&snic->tag_format, sw_tag);
+	tag |= idx << PPOD_IDX_SHIFT;
 
 	hdr.rsvd = 0;
 	hdr.vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid));
-	hdr.pgsz_tag_clr = htonl(tag & snic->tag_format.rsvd_tag_mask);
+	hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask);
 	hdr.maxoffset = htonl(xferlen);
 	hdr.pgoffset = htonl(gl->offset);
 
@@ -372,30 +377,35 @@ free_gl:
 	return RESERVED_ITT;
 }
 
-void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *snic, u32 tag,
-			    struct scatterlist *sgl, unsigned int sgcnt)
+void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *snic, u32 tag)
 {
-	u32 idx = (tag >> snic->tag_format.rsvd_shift) &
-		  snic->tag_format.rsvd_mask;
+	struct cxgb3i_ddp_info *ddp = snic->ddp;
+	u32 idx;
 
-	if (idx < snic->tag_format.rsvd_mask) {
-		struct cxgb3i_ddp_info *ddp = snic->ddp;
+	if (!ddp) {
+		cxgb3i_log_error("release ddp tag 0x%x, ddp NULL.\n", tag);
+		return;
+	}
+
+	idx = (tag >> PPOD_IDX_SHIFT) & ddp->idx_mask;
+	if (idx < ddp->nppods) {
 		struct cxgb3i_gather_list *gl = ddp->gl_map[idx];
 		unsigned int npods;
 
 		if (!gl || !gl->nelem) {
-			cxgb3i_log_warn("release tag 0x%x, idx 0x%x, no gl.\n",
-					tag, idx);
+			cxgb3i_log_error("release tag 0x%x, idx 0x%x, no gl.\n",
+					 tag, idx);
 			return;
 		}
 		npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
-
 		cxgb3i_tag_debug("ddp tag 0x%x, release idx 0x%x, npods %u.\n",
 				 tag, idx, npods);
 		clear_ddp_map(snic, idx, npods);
 		ddp_unmark_entries(ddp, idx, npods);
 		ddp_gl_unmap(snic->pdev, gl);
-	}
+	} else
+		 cxgb3i_log_error("ddp tag 0x%x, idx 0x%x > max 0x%x.\n",
+						  tag, idx, ddp->nppods);
 }
 
 int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *cconn, int hcrc, int dcrc)
@@ -403,12 +413,18 @@ int cxgb3i_conn_ulp_setup(struct cxgb3i_
 	struct iscsi_tcp_conn *tcp_conn = cconn->conn->dd_data;
 	struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
 	struct sk_buff *skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
-					GFP_KERNEL | __GFP_NOFAIL);
+					GFP_KERNEL);
 	struct cpl_set_tcb_field *req;
 	u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
 
+	if (!skb)
+		return -ENOMEM;
+
 	if (page_idx < ULP2_PGIDX_MAX)
 		val |= page_idx << 4;
+	else
+		cxgb3i_log_warn("TID 0x%x, host page 0x%lx default to 4K.\n",
+				c3cn->tid, PAGE_SIZE);
 
 	/* set up ulp submode and page size */
 	req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
@@ -476,14 +492,14 @@ static int cxgb3i_conn_read_pdu_skb(stru
 				   (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) ?
 		    ISCSI_SEGMENT_DGST_ERR : 0;
 		if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
-			cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, "
-					 "ddp'ed, itt 0x%x.\n",
-					 skb, hdr->opcode & ISCSI_OPCODE_MASK,
-					 tcp_conn->in.datalen, hdr->itt);
+			cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, "
+					"ddp'ed, itt 0x%x.\n",
+					skb, hdr->opcode & ISCSI_OPCODE_MASK,
+					tcp_conn->in.datalen, hdr->itt);
 			segment->total_copied = segment->total_size;
 		} else {
-			cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, "
-					 "not ddp'ed, itt 0x%x.\n",
+			cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, "
+					"not ddp'ed, itt 0x%x.\n",
 					 skb, hdr->opcode & ISCSI_OPCODE_MASK,
 					 tcp_conn->in.datalen, hdr->itt);
 			offset += sizeof(struct cpl_iscsi_hdr_norss);
@@ -520,24 +536,141 @@ static inline void tx_skb_setmode(struct
 	skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) | submode;
 }
 
+static int sg_page_coalesce(struct scatterlist *sg, unsigned int offset,
+			    unsigned int dlen, skb_frag_t *frags, int frag_max)
+{
+	unsigned int sglen = sg->length - offset;
+	struct page *page = sg_page(sg);
+	unsigned int datalen = dlen, copy;
+	int i;
+
+	i = 0;
+	do {
+		if (!sglen) {
+			sg = sg_next(sg);
+			offset = 0;
+			sglen = sg->length;
+			page = sg_page(sg);
+		}
+		copy = min(datalen, sglen);
+		if (i && page == frags[i - 1].page &&
+		    offset + sg->offset ==
+			frags[i - 1].page_offset + frags[i - 1].size) {
+			frags[i - 1].size += copy;
+		} else {
+			if (i >= frag_max) {
+				cxgb3i_log_error("%s, too many pages > %u, "
+						 "dlen %u.\n", __func__,
+						 frag_max, dlen);
+				return -EINVAL;
+			}
+
+			frags[i].page = page;
+			frags[i].page_offset = sg->offset + offset;
+			frags[i].size = copy;
+			i++;
+		}
+		datalen -= copy;
+		offset += copy;
+		sglen -= copy;
+	} while (datalen);
+
+	return i;
+}
+
+static int copy_frags_to_skb_pages(struct sk_buff *skb, skb_frag_t *frags,
+				   int frag_cnt, unsigned int datalen)
+{
+	struct page *page = NULL;
+	unsigned char *dp;
+	unsigned int pg_left = 0;
+	unsigned int copy_total = 0;
+	int i;
+
+	for (i = 0; i < frag_cnt; i++, frags++) {
+		while (frags->size) {
+			unsigned char *sp = page_address(frags->page);
+			unsigned int copy;
+
+			if (!pg_left) {
+				int cnt = skb_shinfo(skb)->nr_frags;
+
+				if (cnt >= MAX_SKB_FRAGS) {
+					cxgb3i_log_error("%s: pdu data %u.\n",
+							 __func__, datalen);
+					return -EINVAL;
+				}
+				page = alloc_page(GFP_ATOMIC);
+				if (!page)
+					return -ENOMEM;
+				dp = page_address(page);
+				pg_left = PAGE_SIZE;
+
+				copy = min(pg_left, datalen);
+				skb_fill_page_desc(skb, cnt, page, 0, copy);
+
+				skb->len += copy;
+				skb->data_len += copy;
+				skb->truesize += copy;
+				datalen -= copy;
+			}
+			copy = min(pg_left, frags->size);
+			memcpy(dp, sp + frags->page_offset, copy);
+
+			frags->size -= copy;
+			frags->page_offset += copy;
+			dp += copy;
+			pg_left -= copy;
+			copy_total += copy;
+		}
+	}
+
+	return copy_total;
+}
+
 int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *conn)
 {
-	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+	struct cxgb3i_conn *cconn = conn->dd_data;
+	struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
 	struct iscsi_segment *hdr_seg = &tcp_conn->out.segment;
 	struct iscsi_segment *data_seg = &tcp_conn->out.data_segment;
 	unsigned int hdrlen = hdr_seg->total_size;
 	unsigned int datalen = data_seg->total_size;
 	unsigned int padlen = iscsi_padding(datalen);
-	unsigned int copymax = SKB_MAX_HEAD(TX_HEADER_LEN);
-	unsigned int copylen;
+	unsigned int copylen = hdrlen;
+	unsigned int copy_dlen = 0;
 	struct sk_buff *skb;
 	unsigned char *dst;
+	int i, frag_cnt = 0;
 	int err = -EAGAIN;
 
-	if (data_seg->data && ((datalen + padlen) < copymax))
-		copylen = hdrlen + datalen + padlen;
-	else
-		copylen = hdrlen;
+	/*
+	 * the whole pdu needs to fit into one skb, make sure we don't overrun
+	 * the skb's frag_list. If there are more sg pages than MAX_SKB_FRAGS,
+	 * we have to copy the data either to the head or newly allocated
+	 * whole new page(s). This could happen if the sg contains a lot of
+	 * fragmented data chunks (pages).
+	 */
+	if (datalen) {
+		if (!data_seg->data) {
+			err = sg_page_coalesce(data_seg->sg,
+						data_seg->sg_offset,
+						data_seg->total_size,
+						cconn->frags,
+						TX_PDU_PAGES_MAX);
+			if (err < 0)
+				return err;
+			frag_cnt = err;
+
+			if (frag_cnt > MAX_SKB_FRAGS ||
+			    (padlen && frag_cnt + 1 > MAX_SKB_FRAGS))
+				copy_dlen = datalen + padlen;
+		} else
+			copy_dlen += datalen + padlen;
+	}
+
+	if (copylen + copy_dlen < skb_copymax)
+		copylen += copy_dlen;
 
 	/* supports max. 16K pdus, so one skb is enough to hold all the data */
 	skb = alloc_skb(TX_HEADER_LEN + copylen, GFP_ATOMIC);
@@ -575,70 +708,84 @@ int cxgb3i_conn_ulp2_xmit(struct iscsi_c
 			skb->data_len += datalen;
 			skb->truesize += datalen;
 		}
-	} else {
-		struct scatterlist *sg = data_seg->sg;
-		unsigned int offset = data_seg->sg_offset;
-		struct page *page = sg_page(sg);
-		unsigned int sglen = sg->length - offset;
-
-		do {
-			int i = skb_shinfo(skb)->nr_frags;
-			unsigned int copy;
+	} else if (copy_dlen) {
+		/* need to copy the page fragments */
+		if (copylen > hdrlen) {
+			skb_frag_t *frag = cconn->frags;
 
-			if (!sglen) {
-				sg = sg_next(sg);
-				page = sg_page(sg);
-				offset = 0;
-				sglen = sg->length;
+			/* data fits in the skb's headroom */
+			for (i = 0; i < frag_cnt; i++, frag++) {
+				memcpy(dst,
+					page_address(frag->page) +
+						frag->page_offset,
+					frag->size);
+				dst += frag->size;
 			}
-			copy = min(sglen, datalen);
-
-			if (i && skb_can_coalesce(skb, i, page,
-						  sg->offset + offset)) {
-				skb_shinfo(skb)->frags[i - 1].size += copy;
-			} else {
-				get_page(page);
-				skb_fill_page_desc(skb, i, page,
-						   sg->offset + offset, copy);
+			if (padlen)
+				memset(dst, 0, padlen);
+		} else {
+			/* allocate pages to hold the data */
+			err = copy_frags_to_skb_pages(skb, cconn->frags,
+						      frag_cnt, datalen);
+			if (err < 0) {
+				err = -EAGAIN;
+				goto free_skb;
 			}
-			skb->len += copy;
-			skb->data_len += copy;
-			skb->truesize += copy;
-			offset += copy;
-			sglen -= copy;
-			datalen -= copy;
-		} while (datalen);
-	}
-
-	if (padlen && skb_shinfo(skb)->nr_frags) {
-		int idx = skb_shinfo(skb)->nr_frags;
-		get_page(pad_page);
-		skb_fill_page_desc(skb, idx, pad_page, 0, padlen);
-		skb->data_len += padlen;
-		skb->truesize += padlen;
-		skb->len += padlen;
+			WARN_ON(err != datalen);
+			if (padlen) {
+				skb_frag_t *frag;
+
+				i = skb_shinfo(skb)->nr_frags;
+				frag = &skb_shinfo(skb)->frags[i];
+				dst = page_address(frag->page);
+
+				memset(dst + frag->page_offset + frag->size,
+				       0, padlen);
+				frag->size += padlen;
+			}
+		}
+	} else {
+		/* sg pages fit into frag_list */
+		for (i = 0; i < frag_cnt; i++)
+			get_page(cconn->frags[i].page);
+		memcpy(skb_shinfo(skb)->frags, cconn->frags,
+			sizeof(skb_frag_t) * frag_cnt);
+		skb_shinfo(skb)->nr_frags = frag_cnt;
+		skb->len += datalen;
+		skb->data_len += datalen;
+		skb->truesize += datalen;
+
+		if (padlen) {
+			i = skb_shinfo(skb)->nr_frags;
+			get_page(pad_page);
+			skb_fill_page_desc(skb, i, pad_page, 0, padlen);
+			skb->len += padlen;
+			skb->data_len += padlen;
+			skb->truesize += padlen;
+		}
 	}
 
 send_pdu:
 	err = cxgb3i_c3cn_send_pdus((struct s3_conn *)tcp_conn->sock, skb);
-
 	if (err > 0) {
 		int pdulen = hdrlen + datalen + padlen;
+
 		if (conn->hdrdgst_en)
 			pdulen += ISCSI_DIGEST_SIZE;
 		if (datalen && conn->datadgst_en)
 			pdulen += ISCSI_DIGEST_SIZE;
 
 		hdr_seg->total_copied = hdr_seg->total_size;
-		if (datalen)
-			data_seg->total_copied = data_seg->total_size;
+		data_seg->total_copied = data_seg->total_size;
 		conn->txdata_octets += pdulen;
 		return pdulen;
 	}
 
+free_skb:
 	kfree_skb(skb);
 	if (err < 0 && err != -EAGAIN) {
-		cxgb3i_log_error("conn 0x%p, xmit err %d.\n", conn, err);
+		cxgb3i_log_error("conn 0x%p, xmit err %d, skb len %u/%u.\n",
+				 conn, err, skb->len, skb->data_len);
 		iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
 		return err;
 	}
@@ -652,6 +799,9 @@ int cxgb3i_ulp2_init(void)
 		return -ENOMEM;
 	memset(page_address(pad_page), 0, PAGE_SIZE);
 	cxgb3i_ddp_page_init();
+	cxgb3i_log_info("skb max. frag %u, head %u.\n",
+			(unsigned int)MAX_SKB_FRAGS,
+			(unsigned int)skb_copymax);
 	return 0;
 }
 
@@ -720,7 +870,7 @@ void cxgb3i_conn_closing(struct s3_conn
 
 	read_lock(&c3cn->callback_lock);
 	conn = c3cn->user_data;
-	if (conn && c3cn->state != C3CN_STATE_ESTABLISHED)
+	if (conn)
 		iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
 	read_unlock(&c3cn->callback_lock);
 }
@@ -730,7 +880,7 @@ int cxgb3i_adapter_ulp_init(struct cxgb3
 	struct t3cdev *tdev = snic->tdev;
 	struct cxgb3i_ddp_info *ddp;
 	struct ulp_iscsi_info uinfo;
-	unsigned int ppmax, bits, max_bits;
+	unsigned int ppmax, bits;
 	int i, err;
 
 	err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
@@ -740,26 +890,21 @@ int cxgb3i_adapter_ulp_init(struct cxgb3
 		return err;
 	}
 
-	ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
-	max_bits = min(PPOD_IDX_MAX_SIZE,
-		       (32 - sw_tag_idx_bits - sw_tag_age_bits));
-	bits = __ilog2_u32(ppmax) + 1;
-	if (bits > max_bits)
-		bits = max_bits;
-	ppmax = (1 << bits) - 1;
-
 	snic->tx_max_size = min_t(unsigned int,
 				  uinfo.max_txsz, ULP2_MAX_PKT_SIZE);
 	snic->rx_max_size = min_t(unsigned int,
 				  uinfo.max_rxsz, ULP2_MAX_PKT_SIZE);
+	cxgb3i_log_info("ddp max pkt size: %u/%u,%u, %u/%u,%u.\n",
+			snic->tx_max_size, uinfo.max_txsz, ULP2_MAX_PKT_SIZE,
+			snic->rx_max_size, uinfo.max_rxsz, ULP2_MAX_PKT_SIZE);
 
-	snic->tag_format.idx_bits = sw_tag_idx_bits;
-	snic->tag_format.age_bits = sw_tag_age_bits;
-	snic->tag_format.rsvd_bits = bits;
-	snic->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
-	snic->tag_format.rsvd_mask = (1 << snic->tag_format.rsvd_bits) - 1;
-	snic->tag_format.rsvd_tag_mask =
-		(1 << (snic->tag_format.rsvd_bits + PPOD_IDX_SHIFT)) - 1;
+	snic->tag_format.sw_bits = sw_tag_idx_bits + sw_tag_age_bits;
+
+	ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
+	bits = __ilog2_u32(ppmax) + 1;
+	if (bits > PPOD_IDX_MAX_SIZE)
+		bits = PPOD_IDX_MAX_SIZE;
+	ppmax = (1 << (bits - 1)) - 1;
 
 	ddp = cxgb3i_alloc_big_mem(sizeof(struct cxgb3i_ddp_info) +
 				   ppmax *
@@ -779,12 +924,15 @@ int cxgb3i_adapter_ulp_init(struct cxgb3
 	spin_lock_init(&ddp->map_lock);
 	ddp->llimit = uinfo.llimit;
 	ddp->ulimit = uinfo.ulimit;
+	ddp->nppods = ppmax;
+	ddp->idx_last = ppmax;
+	ddp->idx_bits = bits;
+	ddp->idx_mask = (1 << bits) - 1;
+	ddp->rsvd_tag_mask = (1 << (bits + PPOD_IDX_SHIFT)) - 1;
 
-	uinfo.tagmask =
-	    snic->tag_format.rsvd_mask << snic->tag_format.rsvd_shift;
+	uinfo.tagmask = ddp->idx_mask << PPOD_IDX_SHIFT;
 	for (i = 0; i < ULP2_PGIDX_MAX; i++)
 		uinfo.pgsz_factor[i] = ddp_page_order[i];
-
 	uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
 
 	err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
@@ -794,19 +942,20 @@ int cxgb3i_adapter_ulp_init(struct cxgb3
 		goto free_ppod_map;
 	}
 
-	ddp->nppods = ppmax;
-	ddp->idx_last = ppmax;
-
 	tdev->ulp_iscsi = snic->ddp = ddp;
 
-	cxgb3i_log_info("snic nppods %u (0x%x ~ 0x%x), rsvd shift %u, "
-			"bits %u, mask 0x%x, 0x%x, pkt %u,%u.\n",
-			ppmax, ddp->llimit, ddp->ulimit,
-			snic->tag_format.rsvd_shift,
-			snic->tag_format.rsvd_bits,
-			snic->tag_format.rsvd_mask, uinfo.tagmask,
-			snic->tx_max_size, snic->rx_max_size);
+	cxgb3i_log_info("nppods %u (0x%x ~ 0x%x), bits %u, mask 0x%x,0x%x.\n",
+			ppmax, ddp->llimit, ddp->ulimit, ddp->idx_bits,
+			ddp->idx_mask, ddp->rsvd_tag_mask);
 
+	snic->tag_format.rsvd_bits = ddp->idx_bits;
+	snic->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
+	snic->tag_format.rsvd_mask = (1 << snic->tag_format.rsvd_bits) - 1;
+
+	cxgb3i_log_info("tag format: sw %u, rsvd %u,%u, mask 0x%x.\n",
+			snic->tag_format.sw_bits, snic->tag_format.rsvd_bits,
+			snic->tag_format.rsvd_shift,
+			snic->tag_format.rsvd_mask);
 	return 0;
 
 free_ppod_map:
--- a/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
+++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
@@ -13,7 +13,6 @@
 #ifndef __CXGB3I_ULP2_H__
 #define __CXGB3I_ULP2_H__
 
-#define ULP2_PDU_PAYLOAD_DFLT	(16224 - ISCSI_PDU_HEADER_MAX)
 #define PPOD_PAGES_MAX		4
 #define PPOD_PAGES_SHIFT	2	/* 4 pages per pod */
 
@@ -100,9 +99,6 @@ struct cpl_rx_data_ddp_norss {
 #define ULP2_FLAG_DCRC_ERROR		0x20
 #define ULP2_FLAG_PAD_ERROR		0x40
 
-#define ULP2_MAX_PKT_SIZE		16224
-#define ULP2_MAX_PDU_SIZE		8192
-
 void cxgb3i_conn_closing(struct s3_conn *);
 void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn);
 void cxgb3i_conn_tx_open(struct s3_conn *c3cn);