--- /dev/null
+Subject: fixes bug in tag release and sync-up cxgb3i with mainline state
+From: Karen Xie <kxie@chelsio.com>
+References: 464508 - LTC50816
+
+This patch fixes the tag generation and release and also updates the tcp connection offload to sync-up with the current kernel.org version.
+
+Signed-off-by: Karen Xie <kxie@chelsio.com>
+Signed-off-by: Olaf Hering <olh@suse.de>
+
+---
+ drivers/scsi/cxgb3i/cxgb3i_init.c | 4
+ drivers/scsi/cxgb3i/cxgb3i_iscsi.c | 99 -
+ drivers/scsi/cxgb3i/cxgb3i_offload.c | 2615 +++++++++++++++--------------------
+ drivers/scsi/cxgb3i/cxgb3i_offload.h | 251 +--
+ drivers/scsi/cxgb3i/cxgb3i_ulp2.c | 25
+ drivers/scsi/cxgb3i/cxgb3i_ulp2.h | 23
+ 6 files changed, 1429 insertions(+), 1588 deletions(-)
+
+--- a/drivers/scsi/cxgb3i/cxgb3i_init.c
++++ b/drivers/scsi/cxgb3i/cxgb3i_init.c
+@@ -12,7 +12,7 @@
+ #include "cxgb3i.h"
+
+ #define DRV_MODULE_NAME "cxgb3i"
+-#define DRV_MODULE_VERSION "1.0.0"
++#define DRV_MODULE_VERSION "0.1.0"
+ #define DRV_MODULE_RELDATE "Jun. 1, 2008"
+
+ static char version[] =
+@@ -48,7 +48,6 @@ static void open_s3_dev(struct t3cdev *t
+ vers_printed = 1;
+ }
+
+- cxgb3i_log_debug("open cxgb3 %s.\n", t3dev->name);
+ cxgb3i_sdev_add(t3dev, &t3c_client);
+ cxgb3i_adapter_add(t3dev);
+ }
+@@ -59,7 +58,6 @@ static void open_s3_dev(struct t3cdev *t
+ */
+ static void close_s3_dev(struct t3cdev *t3dev)
+ {
+- cxgb3i_log_debug("close cxgb3 %s.\n", t3dev->name);
+ cxgb3i_adapter_remove(t3dev);
+ cxgb3i_sdev_remove(t3dev);
+ }
+--- a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
++++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
+@@ -24,6 +24,18 @@
+ #include "cxgb3i.h"
+ #include "cxgb3i_ulp2.h"
+
++#ifdef __DEBUG_CXGB3I_TAG__
++#define cxgb3i_tag_debug cxgb3i_log_debug
++#else
++#define cxgb3i_tag_debug(fmt...)
++#endif
++
++#ifdef __DEBUG_CXGB3I_API__
++#define cxgb3i_api_debug cxgb3i_log_debug
++#else
++#define cxgb3i_api_debug(fmt...)
++#endif
++
+ #define align_to_4k_boundary(n) \
+ do { \
+ n = (n) & ~((1 << 12) - 1); \
+@@ -48,7 +60,7 @@ struct cxgb3i_adapter *cxgb3i_adapter_ad
+
+ snic = kzalloc(sizeof(*snic), GFP_KERNEL);
+ if (!snic) {
+- cxgb3i_log_debug("cxgb3 %s, OOM.\n", t3dev->name);
++ cxgb3i_api_debug("cxgb3 %s, OOM.\n", t3dev->name);
+ return NULL;
+ }
+
+@@ -164,7 +176,7 @@ struct cxgb3i_hba *cxgb3i_hba_host_add(s
+ goto pci_dev_put;
+ }
+
+- cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n",
++ cxgb3i_api_debug("shost 0x%p, hba 0x%p, no %u.\n",
+ shost, hba, shost->host_no);
+
+ return hba;
+@@ -177,7 +189,7 @@ pci_dev_put:
+
+ void cxgb3i_hba_host_remove(struct cxgb3i_hba *hba)
+ {
+- cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n",
++ cxgb3i_api_debug("shost 0x%p, hba 0x%p, no %u.\n",
+ hba->shost, hba, hba->shost->host_no);
+ iscsi_host_remove(hba->shost);
+ pci_dev_put(hba->snic->pdev);
+@@ -218,7 +230,7 @@ static struct iscsi_endpoint *cxgb3i_ep_
+ cxgb3i_log_info("NOT going through cxgbi device.\n");
+ goto release_conn;
+ }
+- if (c3cn_in_state(c3cn, C3CN_STATE_CLOSE)) {
++ if (c3cn_is_closing(c3cn)) {
+ err = -ENOSPC;
+ cxgb3i_log_info("ep connect unable to connect.\n");
+ goto release_conn;
+@@ -234,12 +246,12 @@ static struct iscsi_endpoint *cxgb3i_ep_
+ cep->c3cn = c3cn;
+ cep->hba = hba;
+
+- cxgb3i_log_debug("ep 0x%p, 0x%p, c3cn 0x%p, hba 0x%p.\n",
++ cxgb3i_api_debug("ep 0x%p, 0x%p, c3cn 0x%p, hba 0x%p.\n",
+ ep, cep, c3cn, hba);
+ return ep;
+
+ release_conn:
+- cxgb3i_log_debug("conn 0x%p failed, release.\n", c3cn);
++ cxgb3i_api_debug("conn 0x%p failed, release.\n", c3cn);
+ if (c3cn)
+ cxgb3i_c3cn_release(c3cn);
+ return ERR_PTR(err);
+@@ -257,9 +269,9 @@ static int cxgb3i_ep_poll(struct iscsi_e
+ struct cxgb3i_endpoint *cep = ep->dd_data;
+ struct s3_conn *c3cn = cep->c3cn;
+
+- if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED))
++ if (!c3cn_is_established(c3cn))
+ return 0;
+- cxgb3i_log_debug("ep 0x%p, c3cn 0x%p established.\n", ep, c3cn);
++ cxgb3i_api_debug("ep 0x%p, c3cn 0x%p established.\n", ep, c3cn);
+ return 1;
+ }
+
+@@ -274,7 +286,7 @@ static void cxgb3i_ep_disconnect(struct
+ struct cxgb3i_endpoint *cep = ep->dd_data;
+ struct cxgb3i_conn *cconn = cep->cconn;
+
+- cxgb3i_log_debug("ep 0x%p, cep 0x%p.\n", ep, cep);
++ cxgb3i_api_debug("ep 0x%p, cep 0x%p.\n", ep, cep);
+
+ if (cconn && cconn->conn) {
+ struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
+@@ -291,7 +303,7 @@ static void cxgb3i_ep_disconnect(struct
+ write_unlock_bh(&cep->c3cn->callback_lock);
+ }
+
+- cxgb3i_log_debug("ep 0x%p, cep 0x%p, release c3cn 0x%p.\n",
++ cxgb3i_api_debug("ep 0x%p, cep 0x%p, release c3cn 0x%p.\n",
+ ep, cep, cep->c3cn);
+ cxgb3i_c3cn_release(cep->c3cn);
+ iscsi_destroy_endpoint(ep);
+@@ -325,7 +337,7 @@ cxgb3i_session_create(struct iscsi_endpo
+ cep = ep->dd_data;
+ hba = cep->hba;
+ shost = hba->shost;
+- cxgb3i_log_debug("ep 0x%p, cep 0x%p, hba 0x%p.\n", ep, cep, hba);
++ cxgb3i_api_debug("ep 0x%p, cep 0x%p, hba 0x%p.\n", ep, cep, hba);
+ BUG_ON(hba != iscsi_host_priv(shost));
+
+ *host_no = shost->host_no;
+@@ -364,7 +376,7 @@ remove_session:
+ */
+ static void cxgb3i_session_destroy(struct iscsi_cls_session *cls_session)
+ {
+- cxgb3i_log_debug("sess 0x%p.\n", cls_session);
++ cxgb3i_api_debug("sess 0x%p.\n", cls_session);
+ iscsi_r2tpool_free(cls_session->dd_data);
+ iscsi_session_teardown(cls_session);
+ }
+@@ -380,10 +392,10 @@ static inline int cxgb3i_conn_max_xmit_d
+ {
+ struct cxgb3i_conn *cconn = conn->dd_data;
+ unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_SIZE,
+- cconn->hba->snic->tx_max_size -
+- ISCSI_PDU_HEADER_MAX);
++ cconn->hba->snic->tx_max_size -
++ ISCSI_PDU_HEADER_MAX);
+
+- cxgb3i_log_debug("conn 0x%p, max xmit %u.\n",
++ cxgb3i_api_debug("conn 0x%p, max xmit %u.\n",
+ conn, conn->max_xmit_dlength);
+
+ if (conn->max_xmit_dlength)
+@@ -394,7 +406,7 @@ static inline int cxgb3i_conn_max_xmit_d
+
+ align_to_4k_boundary(conn->max_xmit_dlength);
+
+- cxgb3i_log_debug("conn 0x%p, set max xmit %u.\n",
++ cxgb3i_api_debug("conn 0x%p, set max xmit %u.\n",
+ conn, conn->max_xmit_dlength);
+
+ return 0;
+@@ -404,18 +416,18 @@ static inline int cxgb3i_conn_max_recv_d
+ {
+ struct cxgb3i_conn *cconn = conn->dd_data;
+ unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_SIZE,
+- cconn->hba->snic->rx_max_size -
+- ISCSI_PDU_HEADER_MAX);
++ cconn->hba->snic->rx_max_size -
++ ISCSI_PDU_HEADER_MAX);
+
+- cxgb3i_log_debug("conn 0x%p, max recv %u.\n",
++ cxgb3i_api_debug("conn 0x%p, max recv %u.\n",
+ conn, conn->max_recv_dlength);
+
+ align_to_4k_boundary(max);
+
+ if (conn->max_recv_dlength) {
+ if (conn->max_recv_dlength > max) {
+- cxgb3i_log_error("MaxRecvDataSegmentLength %u, not supported."
+- "Need to be <= %u.\n",
++ cxgb3i_log_error("MaxRecvDataSegmentLength %u, not "
++ "supported. Need to be <= %u.\n",
+ conn->max_recv_dlength, max);
+ return -EINVAL;
+ }
+@@ -425,7 +437,7 @@ static inline int cxgb3i_conn_max_recv_d
+ } else
+ conn->max_recv_dlength = max;
+
+- cxgb3i_log_debug("conn 0x%p, set max recv %u.\n",
++ cxgb3i_api_debug("conn 0x%p, set max recv %u.\n",
+ conn, conn->max_recv_dlength);
+
+ return 0;
+@@ -438,7 +450,7 @@ static struct iscsi_cls_conn *cxgb3i_con
+ struct iscsi_conn *conn;
+ struct cxgb3i_conn *cconn;
+
+- cxgb3i_log_debug("sess 0x%p, cid %u.\n", cls_session, cid);
++ cxgb3i_api_debug("sess 0x%p, cid %u.\n", cls_session, cid);
+
+ cls_conn = iscsi_conn_setup(cls_session, sizeof(*cconn), cid);
+ if (!cls_conn)
+@@ -495,7 +507,7 @@ static int cxgb3i_conn_bind(struct iscsi
+ if (!ep)
+ return -EINVAL;
+
+- cxgb3i_log_debug("ep 0x%p, cls sess 0x%p, cls conn 0x%p.\n",
++ cxgb3i_api_debug("ep 0x%p, cls sess 0x%p, cls conn 0x%p.\n",
+ ep, cls_session, cls_conn);
+
+ err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
+@@ -544,7 +556,7 @@ static int cxgb3i_conn_get_param(struct
+ struct iscsi_conn *conn = cls_conn->dd_data;
+ int len;
+
+- cxgb3i_log_debug("cls_conn 0x%p, param %d.\n", cls_conn, param);
++ cxgb3i_api_debug("cls_conn 0x%p, param %d.\n", cls_conn, param);
+
+ switch (param) {
+ case ISCSI_PARAM_CONN_PORT:
+@@ -621,15 +633,22 @@ static int cxgb3i_host_set_param(struct
+ {
+ struct cxgb3i_hba *hba = iscsi_host_priv(shost);
+
+- cxgb3i_log_debug("param %d, buf %s.\n", param, buf);
++ cxgb3i_api_debug("param %d, buf %s.\n", param, buf);
+
+- if (hba && param == ISCSI_HOST_PARAM_IPADDRESS) {
++ switch (param) {
++ case ISCSI_HOST_PARAM_IPADDRESS:
++ {
+ __be32 addr = in_aton(buf);
+ cxgb3i_set_private_ipv4addr(hba->ndev, addr);
+ return 0;
+ }
+-
+- return iscsi_host_get_param(shost, param, buf);
++ case ISCSI_HOST_PARAM_HWADDRESS:
++ case ISCSI_HOST_PARAM_NETDEV_NAME:
++ /* ignore */
++ return 0;
++ default:
++ return iscsi_host_set_param(shost, param, buf, buflen);
++ }
+ }
+
+ /**
+@@ -645,7 +664,7 @@ static int cxgb3i_host_get_param(struct
+ int i;
+ int len = 0;
+
+- cxgb3i_log_debug("hba %s, param %d.\n", hba->ndev->name, param);
++ cxgb3i_api_debug("hba %s, param %d.\n", hba->ndev->name, param);
+
+ switch (param) {
+ case ISCSI_HOST_PARAM_HWADDRESS:
+@@ -720,6 +739,10 @@ static inline void cxgb3i_parse_tag(stru
+ << format->rsvd_shift;
+ *sw_bits |= tag & ((1 << format->rsvd_shift) - 1);
+ }
++
++ cxgb3i_tag_debug("parse tag 0x%x, rsvd 0x%x, sw 0x%x.\n",
++ tag, rsvd_bits ? *rsvd_bits : 0xFFFFFFFF,
++ sw_bits ? *sw_bits : 0xFFFFFFFF);
+ }
+
+
+@@ -735,6 +758,9 @@ static void cxgb3i_parse_itt(struct iscs
+ *idx = sw_bits & ISCSI_ITT_MASK;
+ if (age)
+ *age = (sw_bits >> snic->tag_format.idx_bits) & ISCSI_AGE_MASK;
++
++ cxgb3i_tag_debug("parse itt 0x%x, idx 0x%x, age 0x%x.\n",
++ itt, idx ? *idx : 0xFFFFF, age ? *age : 0xFF);
+ }
+
+ static int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
+@@ -762,6 +788,9 @@ static int cxgb3i_reserve_itt(struct isc
+ snic->tag_format.rsvd_shift);
+ *hdr_itt = htonl(tag);
+
++ cxgb3i_tag_debug("new tag 0x%x/0x%x (itt 0x%x, age 0x%x).\n",
++ tag, *hdr_itt, task->itt, sess->age);
++
+ return 0;
+ }
+
+@@ -771,10 +800,12 @@ static void cxgb3i_release_itt(struct is
+ struct iscsi_conn *conn = task->conn;
+ struct cxgb3i_conn *cconn = conn->dd_data;
+ struct cxgb3i_adapter *snic = cconn->hba->snic;
++ u32 tag = ntohl(hdr_itt);
++
++ cxgb3i_tag_debug("release tag 0x%x.\n", tag);
+
+- hdr_itt = ntohl(hdr_itt);
+ if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE))
+- cxgb3i_ddp_tag_release(snic, hdr_itt,
++ cxgb3i_ddp_tag_release(snic, tag,
+ scsi_in(sc)->table.sgl,
+ scsi_in(sc)->table.nents);
+ }
+@@ -871,14 +902,14 @@ int cxgb3i_iscsi_init(void)
+ cxgb3i_log_error("Could not register cxgb3i transport.\n");
+ return -ENODEV;
+ }
+- cxgb3i_log_debug("cxgb3i transport 0x%p.\n", cxgb3i_scsi_transport);
++ cxgb3i_api_debug("cxgb3i transport 0x%p.\n", cxgb3i_scsi_transport);
+ return 0;
+ }
+
+ void cxgb3i_iscsi_cleanup(void)
+ {
+ if (cxgb3i_scsi_transport) {
+- cxgb3i_log_debug("cxgb3i transport 0x%p.\n",
++ cxgb3i_api_debug("cxgb3i transport 0x%p.\n",
+ cxgb3i_scsi_transport);
+ iscsi_unregister_transport(&cxgb3i_iscsi_transport);
+ }
+--- a/drivers/scsi/cxgb3i/cxgb3i_offload.c
++++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
+@@ -1,12 +1,15 @@
+ /*
+- * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
++ * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
+ *
+- * Written by Dimitris Michailidis (dm@chelsio.com)
++ * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
+ * release for licensing terms and conditions.
++ *
++ * Written by: Dimitris Michailidis (dm@chelsio.com)
++ * Karen Xie (kxie@chelsio.com)
+ */
+
+ #include <linux/if_vlan.h>
+@@ -18,6 +21,27 @@
+ #include "cxgb3i_offload.h"
+ #include "cxgb3i_ulp2.h"
+
++#ifdef __DEBUG_C3CN_CONN__
++#define c3cn_conn_debug cxgb3i_log_debug
++#else
++#define c3cn_conn_debug(fmt...)
++#endif
++
++#ifdef __DEBUG_C3CN_TX__
++#define c3cn_tx_debug cxgb3i_log_debug
++#else
++#define c3cn_tx_debug(fmt...)
++#endif
++
++#ifdef __DEBUG_C3CN_RX__
++#define c3cn_rx_debug cxgb3i_log_debug
++#else
++#define c3cn_rx_debug(fmt...)
++#endif
++
++/*
++ * module parameters releated to offloaded iscsi connection
++ */
+ static int cxgb3_rcv_win = 256 * 1024;
+ module_param(cxgb3_rcv_win, int, 0644);
+ MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
+@@ -39,30 +63,91 @@ static unsigned int cxgb3_sport_base = 2
+ module_param(cxgb3_sport_base, uint, 0644);
+ MODULE_PARM_DESC(cxgb3_sport_base, "starting port number (default=20000)");
+
+-#ifdef __DEBUG_C3CN_TX__
+-#define c3cn_tx_debug cxgb3i_log_debug
+-#else
+-#define c3cn_tx_debug(fmt...)
+-#endif
++/*
++ * cxgb3i tcp connection data(per adapter) list
++ */
++static LIST_HEAD(cdata_list);
++static DEFINE_RWLOCK(cdata_rwlock);
+
+-#ifdef __DEBUG_C3CN_RX__
+-#define c3cn_rx_debug cxgb3i_log_debug
+-#else
+-#define c3cn_rx_debug(fmt...)
+-#endif
++static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion);
++static void c3cn_release_offload_resources(struct s3_conn *c3cn);
++
++/*
++ * iscsi source port management
++ *
++ * Find a free source port in the port allocation map. We use a very simple
++ * rotor scheme to look for the next free port.
++ *
++ * If a source port has been specified make sure that it doesn't collide with
++ * our normal source port allocation map. If it's outside the range of our
++ * allocation/deallocation scheme just let them use it.
++ *
++ * If the source port is outside our allocation range, the caller is
++ * responsible for keeping track of their port usage.
++ */
++static int c3cn_get_port(struct s3_conn *c3cn, struct cxgb3i_sdev_data *cdata)
++{
++ unsigned int start;
++ int idx;
++
++ if (!cdata)
++ goto error_out;
++
++ if (c3cn->saddr.sin_port != 0) {
++ idx = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
++ if (idx < 0 || idx >= cxgb3_max_connect)
++ return 0;
++ if (!test_and_set_bit(idx, cdata->sport_map))
++ return -EADDRINUSE;
++ }
++
++ /* the sport_map_next may not be accurate but that is okay, sport_map
++ should be */
++ start = idx = cdata->sport_map_next;
++ do {
++ if (++idx >= cxgb3_max_connect)
++ idx = 0;
++ if (!(test_and_set_bit(idx, cdata->sport_map))) {
++ c3cn->saddr.sin_port = htons(cxgb3_sport_base + idx);
++ cdata->sport_map_next = idx;
++ c3cn_conn_debug("%s reserve port %u.\n",
++ cdata->cdev->name,
++ cxgb3_sport_base + idx);
++ return 0;
++ }
++ } while (idx != start);
++
++error_out:
++ return -EADDRNOTAVAIL;
++}
++
++static void c3cn_put_port(struct s3_conn *c3cn)
++{
++ struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(c3cn->cdev);
++
++ if (c3cn->saddr.sin_port) {
++ int idx = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
++
++ c3cn->saddr.sin_port = 0;
++ if (idx < 0 || idx >= cxgb3_max_connect)
++ return;
++ clear_bit(idx, cdata->sport_map);
++ c3cn_conn_debug("%s, release port %u.\n",
++ cdata->cdev->name, cxgb3_sport_base + idx);
++ }
++}
+
+-/* connection flags */
+ static inline void c3cn_set_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
+ {
+ __set_bit(flag, &c3cn->flags);
+- c3cn_conn_debug("c3cn 0x%p, set %d, s 0x%x, f 0x%lx.\n",
++ c3cn_conn_debug("c3cn 0x%p, set %d, s %u, f 0x%lx.\n",
+ c3cn, flag, c3cn->state, c3cn->flags);
+ }
+
+-static inline void c3cn_reset_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
++static inline void c3cn_clear_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
+ {
+ __clear_bit(flag, &c3cn->flags);
+- c3cn_conn_debug("c3cn 0x%p, clear %d, s 0x%x, f 0x%lx.\n",
++ c3cn_conn_debug("c3cn 0x%p, clear %d, s %u, f 0x%lx.\n",
+ c3cn, flag, c3cn->state, c3cn->flags);
+ }
+
+@@ -73,14 +158,12 @@ static inline int c3cn_flag(struct s3_co
+ return test_bit(flag, &c3cn->flags);
+ }
+
+-/* connection state */
+ static void c3cn_set_state(struct s3_conn *c3cn, int state)
+ {
+- c3cn_conn_debug("c3cn 0x%p state -> 0x%x.\n", c3cn, state);
++ c3cn_conn_debug("c3cn 0x%p state -> %u.\n", c3cn, state);
+ c3cn->state = state;
+ }
+
+-/* connection reference count */
+ static inline void c3cn_hold(struct s3_conn *c3cn)
+ {
+ atomic_inc(&c3cn->refcnt);
+@@ -89,432 +172,316 @@ static inline void c3cn_hold(struct s3_c
+ static inline void c3cn_put(struct s3_conn *c3cn)
+ {
+ if (atomic_dec_and_test(&c3cn->refcnt)) {
+- c3cn_conn_debug("free c3cn 0x%p, 0x%x, 0x%lx.\n",
++ c3cn_conn_debug("free c3cn 0x%p, s %u, f 0x%lx.\n",
+ c3cn, c3cn->state, c3cn->flags);
+ kfree(c3cn);
+ }
+ }
+
+-/* minimal port allocation management scheme */
+-static spinlock_t sport_map_lock;
+-static unsigned int sport_map_next;
+-static unsigned long *sport_map;
++static void c3cn_closed(struct s3_conn *c3cn)
++{
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ c3cn_put_port(c3cn);
++ c3cn_release_offload_resources(c3cn);
++ c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
++ cxgb3i_conn_closing(c3cn);
++}
++
++/*
++ * CPL (Chelsio Protocol Language) defines a message passing interface between
++ * the host driver and T3 asic.
++ * The section below implments CPLs that related to iscsi tcp connection
++ * open/close/abort and data send/receive.
++ */
+
+ /*
+- * Find a free source port in our allocation map. We use a very simple rotor
+- * scheme to look for the next free port.
+- *
+- * If a source port has been specified make sure that it doesn't collide with
+- * our normal source port allocation map. If it's outside the range of our
+- * allocation scheme just let them use it.
++ * CPL connection active open request: host ->
+ */
+-static int c3cn_get_port(struct s3_conn *c3cn)
++static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
+ {
+- unsigned int start;
++ int i = 0;
+
+- if (!sport_map)
+- goto error_out;
++ while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
++ ++i;
++ return i;
++}
+
+- if (c3cn->saddr.sin_port != 0) {
+- int sport = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
+- int err = 0;
++static unsigned int select_mss(struct s3_conn *c3cn, unsigned int pmtu)
++{
++ unsigned int idx;
++ struct dst_entry *dst = c3cn->dst_cache;
++ struct t3cdev *cdev = c3cn->cdev;
++ const struct t3c_data *td = T3C_DATA(cdev);
++ u16 advmss = dst_metric(dst, RTAX_ADVMSS);
+
+- if (sport < 0 || sport >= cxgb3_max_connect)
+- return 0;
+- spin_lock(&sport_map_lock);
+- err = __test_and_set_bit(sport, sport_map);
+- spin_unlock(&sport_map_lock);
+- return err ? -EADDRINUSE : 0;
+- }
++ if (advmss > pmtu - 40)
++ advmss = pmtu - 40;
++ if (advmss < td->mtus[0] - 40)
++ advmss = td->mtus[0] - 40;
++ idx = find_best_mtu(td, advmss + 40);
++ return idx;
++}
+
+- spin_lock(&sport_map_lock);
+- start = sport_map_next;
+- do {
+- unsigned int new = sport_map_next;
+- if (++sport_map_next >= cxgb3_max_connect)
+- sport_map_next = 0;
+- if (!(__test_and_set_bit(new, sport_map))) {
+- spin_unlock(&sport_map_lock);
+- c3cn_conn_debug("reserve port %u.\n",
+- cxgb3_sport_base + new);
+- c3cn->saddr.sin_port = htons(cxgb3_sport_base + new);
+- return 0;
+- }
+- } while (sport_map_next != start);
+- spin_unlock(&sport_map_lock);
++static inline int compute_wscale(int win)
++{
++ int wscale = 0;
++ while (wscale < 14 && (65535<<wscale) < win)
++ wscale++;
++ return wscale;
++}
+
+-error_out:
+- return -EADDRNOTAVAIL;
++static inline unsigned int calc_opt0h(struct s3_conn *c3cn)
++{
++ int wscale = compute_wscale(cxgb3_rcv_win);
++ return V_KEEP_ALIVE(1) |
++ F_TCAM_BYPASS |
++ V_WND_SCALE(wscale) |
++ V_MSS_IDX(c3cn->mss_idx);
+ }
+
+-/*
+- * Deallocate a source port from the allocation map. If the source port is
+- * outside our allocation range just return -- the caller is responsible for
+- * keeping track of their port usage outside of our allocation map.
+- */
+-static void c3cn_put_port(struct s3_conn *c3cn)
++static inline unsigned int calc_opt0l(struct s3_conn *c3cn)
+ {
+- if (c3cn->saddr.sin_port) {
+- int old = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
+- c3cn->saddr.sin_port = 0;
++ return V_ULP_MODE(ULP_MODE_ISCSI) |
++ V_RCV_BUFSIZ(cxgb3_rcv_win>>10);
++}
+
+- if (old < 0 || old >= cxgb3_max_connect)
+- return;
++static void make_act_open_req(struct s3_conn *c3cn, struct sk_buff *skb,
++ unsigned int atid, const struct l2t_entry *e)
++{
++ struct cpl_act_open_req *req;
+
+- c3cn_conn_debug("release port %u.\n", cxgb3_sport_base + old);
+- spin_lock(&sport_map_lock);
+- __clear_bit(old, sport_map);
+- spin_unlock(&sport_map_lock);
+- }
++ c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn, atid);
++
++ skb->priority = CPL_PRIORITY_SETUP;
++ req = (struct cpl_act_open_req *)__skb_put(skb, sizeof(*req));
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
++ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
++ req->local_port = c3cn->saddr.sin_port;
++ req->peer_port = c3cn->daddr.sin_port;
++ req->local_ip = c3cn->saddr.sin_addr.s_addr;
++ req->peer_ip = c3cn->daddr.sin_addr.s_addr;
++ req->opt0h = htonl(calc_opt0h(c3cn) | V_L2T_IDX(e->idx) |
++ V_TX_CHANNEL(e->smt_idx));
++ req->opt0l = htonl(calc_opt0l(c3cn));
++ req->params = 0;
+ }
+
+-static void c3cn_reset_timer(struct s3_conn *c3cn, struct timer_list *timer,
+- unsigned long expires)
++static void fail_act_open(struct s3_conn *c3cn, int errno)
+ {
+- if (!mod_timer(timer, expires))
+- c3cn_hold(c3cn);
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++ c3cn->err = errno;
++ c3cn_closed(c3cn);
+ }
+
+-typedef int (cxgb3_cpl_handler_decl) (struct t3cdev *,
+- struct sk_buff *, void *);
+-
+-static cxgb3_cpl_handler_decl do_act_establish;
+-static cxgb3_cpl_handler_decl do_act_open_rpl;
+-static cxgb3_cpl_handler_decl do_wr_ack;
+-static cxgb3_cpl_handler_decl do_peer_close;
+-static cxgb3_cpl_handler_decl do_abort_req;
+-static cxgb3_cpl_handler_decl do_abort_rpl;
+-static cxgb3_cpl_handler_decl do_close_con_rpl;
+-static cxgb3_cpl_handler_decl do_iscsi_hdr;
++static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
++{
++ struct s3_conn *c3cn = (struct s3_conn *)skb->sk;
+
+-static LIST_HEAD(cxgb3_list);
+-static DEFINE_MUTEX(cxgb3_list_lock);
++ c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
+
+-/*
+- * For ULP connections HW may inserts digest bytes into the pdu. This array
+- * contains the compensating extra lengths for ULP packets. It is indexed by
+- * a packet's ULP submode.
+- */
+-static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 };
++ c3cn_hold(c3cn);
++ spin_lock_bh(&c3cn->lock);
++ if (c3cn->state == C3CN_STATE_CONNECTING)
++ fail_act_open(c3cn, EHOSTUNREACH);
++ spin_unlock_bh(&c3cn->lock);
++ c3cn_put(c3cn);
++ __kfree_skb(skb);
++}
+
+ /*
+- * Return the length of any HW additions that will be made to a Tx packet.
+- * Such additions can happen for some types of ULP packets.
++ * CPL connection close request: host ->
++ *
++ * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
++ * the write queue (i.e., after any unsent txt data).
+ */
+-static inline unsigned int ulp_extra_len(const struct sk_buff *skb)
++static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
++ int flags)
+ {
+- return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3];
++ CXGB3_SKB_CB(skb)->seq = c3cn->write_seq;
++ CXGB3_SKB_CB(skb)->flags = flags;
++ __skb_queue_tail(&c3cn->write_queue, skb);
+ }
+
+-/*
+- * Size of WRs in bytes. Note that we assume all devices we are handling have
+- * the same WR size.
+- */
+-static unsigned int wrlen __read_mostly;
+-
+-/*
+- * The number of WRs needed for an skb depends on the number of page fragments
+- * in the skb and whether it has any payload in its main body. This maps the
+- * length of the gather list represented by an skb into the # of necessary WRs.
+- */
+-#define SKB_WR_LIST_SIZE (16384/512 + 1)
+-static unsigned int skb_wrs[SKB_WR_LIST_SIZE + 2] __read_mostly;
+-
+-static void s3_init_wr_tab(unsigned int wr_len)
++static void send_close_req(struct s3_conn *c3cn)
+ {
+- int i;
++ struct sk_buff *skb = c3cn->cpl_close;
++ struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
++ unsigned int tid = c3cn->tid;
+
+- if (skb_wrs[1]) /* already initialized */
+- return;
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
+
+- for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
+- int sgl_len = (3 * i) / 2 + (i & 1);
++ c3cn->cpl_close = NULL;
+
+- sgl_len += 3;
+- skb_wrs[i] = (sgl_len <= wr_len
+- ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
+- }
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
++ req->wr.wr_lo = htonl(V_WR_TID(tid));
++ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
++ req->rsvd = htonl(c3cn->write_seq);
+
+- wrlen = wr_len * 8;
++ skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND);
++ if (c3cn->state != C3CN_STATE_CONNECTING)
++ c3cn_push_tx_frames(c3cn, 1);
+ }
+
+ /*
+- * cxgb3i API operations.
+- */
+-/*
+- * large memory chunk allocation/release
++ * CPL connection abort request: host ->
++ *
++ * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
++ * for the same connection and also that we do not try to send a message
++ * after the connection has closed.
+ */
+-void *cxgb3i_alloc_big_mem(unsigned int size)
++static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb)
+ {
+- void *p = kmalloc(size, GFP_KERNEL);
+- if (!p)
+- p = vmalloc(size);
+- if (p)
+- memset(p, 0, size);
+- return p;
+-}
++ struct cpl_abort_req *req = cplhdr(skb);
+
+-void cxgb3i_free_big_mem(void *addr)
+-{
+- if (is_vmalloc_addr(addr))
+- vfree(addr);
+- else
+- kfree(addr);
+-}
++ c3cn_conn_debug("tdev 0x%p.\n", cdev);
+
+-void cxgb3i_sdev_cleanup(void)
+-{
+- if (sport_map)
+- cxgb3i_free_big_mem(sport_map);
++ req->cmd = CPL_ABORT_NO_RST;
++ cxgb3_ofld_send(cdev, skb);
+ }
+
+-int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers)
++static inline void c3cn_purge_write_queue(struct s3_conn *c3cn)
+ {
+- cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish;
+- cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl;
+- cpl_handlers[CPL_PEER_CLOSE] = do_peer_close;
+- cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req;
+- cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl;
+- cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl;
+- cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack;
+- cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr;
++ struct sk_buff *skb;
+
+- if (cxgb3_max_connect > CXGB3I_MAX_CONN)
+- cxgb3_max_connect = CXGB3I_MAX_CONN;
+- sport_map = cxgb3i_alloc_big_mem(DIV_ROUND_UP(cxgb3_max_connect,
+- 8 *
+- sizeof(unsigned long)));
+- if (!sport_map)
+- return -ENOMEM;
+- return 0;
++ while ((skb = __skb_dequeue(&c3cn->write_queue)))
++ __kfree_skb(skb);
+ }
+
+-void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client)
++static void send_abort_req(struct s3_conn *c3cn)
+ {
+- struct cxgb3i_sdev_data *cdata;
+- struct adap_ports *ports;
+- struct ofld_page_info rx_page_info;
+- unsigned int wr_len;
+- int i;
++ struct sk_buff *skb = c3cn->cpl_abort_req;
++ struct cpl_abort_req *req;
++ unsigned int tid = c3cn->tid;
+
+- cdata = kzalloc(sizeof *cdata, GFP_KERNEL);
+- if (!cdata)
++ if (unlikely(c3cn->state == C3CN_STATE_ABORTING) || !skb ||
++ !c3cn->cdev)
+ return;
+- ports = kzalloc(sizeof *ports, GFP_KERNEL);
+- if (!ports)
+- goto free_ports;
+- cdata->ports = ports;
+-
+- if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 ||
+- cdev->ctl(cdev, GET_PORTS, cdata->ports) < 0 ||
+- cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0)
+- goto free_ports;
+
+- s3_init_wr_tab(wr_len);
++ c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
+
+- INIT_LIST_HEAD(&cdata->list);
+- cdata->cdev = cdev;
+- cdata->client = client;
+- cdata->rx_page_size = rx_page_info.page_size;
+- skb_queue_head_init(&cdata->deferq);
++ c3cn_conn_debug("c3cn 0x%p, flag ABORT_RPL + ABORT_SHUT.\n", c3cn);
+
+- for (i = 0; i < ports->nports; i++)
+- NDEV2CDATA(ports->lldevs[i]) = cdata;
++ c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING);
+
+- mutex_lock(&cxgb3_list_lock);
+- list_add_tail(&cdata->list, &cxgb3_list);
+- mutex_unlock(&cxgb3_list_lock);
++ /* Purge the send queue so we don't send anything after an abort. */
++ c3cn_purge_write_queue(c3cn);
+
+- return;
++ c3cn->cpl_abort_req = NULL;
++ req = (struct cpl_abort_req *)skb->head;
+
+-free_ports:
+- kfree(ports);
+- kfree(cdata);
+-}
++ skb->priority = CPL_PRIORITY_DATA;
++ set_arp_failure_handler(skb, abort_arp_failure);
+
+-void cxgb3i_sdev_remove(struct t3cdev *cdev)
+-{
+- struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
+- struct adap_ports *ports = cdata->ports;
+- int i;
+-
+- for (i = 0; i < ports->nports; i++)
+- NDEV2CDATA(ports->lldevs[i]) = NULL;
+-
+- mutex_lock(&cxgb3_list_lock);
+- list_del(&cdata->list);
+- mutex_unlock(&cxgb3_list_lock);
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
++ req->wr.wr_lo = htonl(V_WR_TID(tid));
++ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
++ req->rsvd0 = htonl(c3cn->snd_nxt);
++ req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT);
++ req->cmd = CPL_ABORT_SEND_RST;
+
+- kfree(ports);
+- kfree(cdata);
++ l2t_send(c3cn->cdev, skb, c3cn->l2t);
+ }
+
+ /*
+- * Return TRUE if the specified net device is for a port on one of our
+- * registered adapters.
++ * CPL connection abort reply: host ->
++ *
++ * Send an ABORT_RPL message in response of the ABORT_REQ received.
+ */
+-static int is_cxgb3_dev(struct net_device *dev)
++static void send_abort_rpl(struct s3_conn *c3cn, int rst_status)
+ {
+- struct cxgb3i_sdev_data *cdata;
++ struct sk_buff *skb = c3cn->cpl_abort_rpl;
++ struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
+
+- mutex_lock(&cxgb3_list_lock);
+- list_for_each_entry(cdata, &cxgb3_list, list) {
+- struct adap_ports *ports = cdata->ports;
+- int i;
++ c3cn->cpl_abort_rpl = NULL;
+
+- for (i = 0; i < ports->nports; i++)
+- if (dev == ports->lldevs[i]) {
+- mutex_unlock(&cxgb3_list_lock);
+- return 1;
+- }
+- }
+- mutex_unlock(&cxgb3_list_lock);
+- return 0;
++ skb->priority = CPL_PRIORITY_DATA;
++ rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
++ rpl->wr.wr_lo = htonl(V_WR_TID(c3cn->tid));
++ OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, c3cn->tid));
++ rpl->cmd = rst_status;
++
++ cxgb3_ofld_send(c3cn->cdev, skb);
+ }
+
+ /*
+- * Primary cxgb3 API operations.
+- * =============================
++ * CPL connection rx data ack: host ->
++ * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
++ * credits sent.
+ */
+-
+-static int s3_push_frames(struct s3_conn *, int);
+-static int s3_send_reset(struct s3_conn *, int, struct sk_buff *);
+-static void t3_release_offload_resources(struct s3_conn *);
+-static void mk_close_req(struct s3_conn *);
+-
+-struct s3_conn *cxgb3i_c3cn_create(void)
++static u32 send_rx_credits(struct s3_conn *c3cn, u32 credits, u32 dack)
+ {
+- struct s3_conn *c3cn;
+-
+- c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL);
+- if (c3cn == NULL)
+- return NULL;
+-
+- c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn);
++ struct sk_buff *skb;
++ struct cpl_rx_data_ack *req;
+
+- c3cn->flags = 0;
+- spin_lock_init(&c3cn->lock);
+- atomic_set(&c3cn->refcnt, 1);
+- skb_queue_head_init(&c3cn->receive_queue);
+- skb_queue_head_init(&c3cn->write_queue);
+- setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn);
+- rwlock_init(&c3cn->callback_lock);
++ skb = alloc_skb(sizeof(*req), GFP_ATOMIC);
++ if (!skb)
++ return 0;
+
+- return c3cn;
++ req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
++ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, c3cn->tid));
++ req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
++ skb->priority = CPL_PRIORITY_ACK;
++ cxgb3_ofld_send(c3cn->cdev, skb);
++ return credits;
+ }
+
+-static inline void s3_purge_write_queue(struct s3_conn *c3cn)
+-{
+- struct sk_buff *skb;
+-
+- while ((skb = __skb_dequeue(&c3cn->write_queue)))
+- __kfree_skb(skb);
+-}
++/*
++ * CPL connection tx data: host ->
++ *
++ * Send iscsi PDU via TX_DATA CPL message. Returns the number of
++ * credits sent.
++ * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
++ * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
++ */
+
+-static void c3cn_done(struct s3_conn *c3cn)
++/*
++ * For ULP connections HW may inserts digest bytes into the pdu. Those digest
++ * bytes are not sent by the host but are part of the TCP payload and therefore
++ * consume TCP sequence space.
++ */
++static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 };
++static inline unsigned int ulp_extra_len(const struct sk_buff *skb)
+ {
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
+-
+- c3cn_put_port(c3cn);
+- t3_release_offload_resources(c3cn);
+- c3cn_set_state(c3cn, C3CN_STATE_CLOSE);
+- c3cn->shutdown = C3CN_SHUTDOWN_MASK;
+- cxgb3i_conn_closing(c3cn);
++ return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3];
+ }
+
+-static void c3cn_close(struct s3_conn *c3cn)
+-{
+- int data_lost, old_state;
+-
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
+-
+- dst_confirm(c3cn->dst_cache);
+-
+- spin_lock_bh(&c3cn->lock);
+- c3cn->shutdown |= C3CN_SHUTDOWN_MASK;
+-
+- /*
+- * We need to flush the receive buffs. We do this only on the
+- * descriptor close, not protocol-sourced closes, because the
+- * reader process may not have drained the data yet! Make a note
+- * of whether any received data will be lost so we can decide whether
+- * to FIN or RST.
+- */
+- data_lost = skb_queue_len(&c3cn->receive_queue);
+- __skb_queue_purge(&c3cn->receive_queue);
+-
+- if (c3cn->state == C3CN_STATE_CLOSE)
+- /* Nothing if we are already closed */
+- c3cn_conn_debug("c3cn 0x%p, 0x%x, already closed.\n",
+- c3cn, c3cn->state);
+- else if (data_lost || c3cn->state == C3CN_STATE_SYN_SENT) {
+- c3cn_conn_debug("c3cn 0x%p, 0x%x -> closing, send reset.\n",
+- c3cn, c3cn->state);
+- /* Unread data was tossed, zap the connection. */
+- s3_send_reset(c3cn, CPL_ABORT_SEND_RST, NULL);
+- goto unlock;
+- } else if (c3cn->state == C3CN_STATE_ESTABLISHED) {
+- c3cn_conn_debug("c3cn 0x%p, est. -> closing, send close_req.\n",
+- c3cn);
+- c3cn_set_state(c3cn, C3CN_STATE_CLOSING);
+- mk_close_req(c3cn);
+- }
+-
+-unlock:
+- old_state = c3cn->state;
+- c3cn_hold(c3cn); /* must last past the potential destroy() */
++static unsigned int wrlen __read_mostly;
+
+- spin_unlock_bh(&c3cn->lock);
++/*
++ * The number of WRs needed for an skb depends on the number of fragments
++ * in the skb and whether it has any payload in its main body. This maps the
++ * length of the gather list represented by an skb into the # of necessary WRs.
++ *
++ * The max. length of an skb is controlled by the max pdu size which is ~16K.
++ * Also, assume the min. fragment length is the sector size (512), then add
++ * extra fragment counts for iscsi bhs and payload padding.
++ */
++#define SKB_WR_LIST_SIZE (16384/512 + 3)
++static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
+
+- /*
+- * There are no more user references at this point. Grab the
+- * connection lock and finish the close.
+- */
+- local_bh_disable();
+- spin_lock(&c3cn->lock);
++static void s3_init_wr_tab(unsigned int wr_len)
++{
++ int i;
+
+- /*
+- * Because the connection was orphaned before the spin_lock()
+- * either the backlog or a BH may have already destroyed it.
+- * Bail out if so.
+- */
+- if (old_state != C3CN_STATE_CLOSE && c3cn->state == C3CN_STATE_CLOSE)
+- goto out;
++ if (skb_wrs[1]) /* already initialized */
++ return;
+
+- if (c3cn->state == C3CN_STATE_CLOSE)
+- s3_purge_write_queue(c3cn);
++ for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
++ int sgl_len = (3 * i) / 2 + (i & 1);
+
+-out:
+- spin_unlock(&c3cn->lock);
+- local_bh_enable();
+- c3cn_put(c3cn);
+-}
++ sgl_len += 3;
++ skb_wrs[i] = (sgl_len <= wr_len
++ ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
++ }
+
+-void cxgb3i_c3cn_release(struct s3_conn *c3cn)
+-{
+- c3cn_conn_debug("c3cn 0x%p, s 0x%x, f 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
+- if (likely(c3cn->state != C3CN_STATE_SYN_SENT))
+- c3cn_close(c3cn);
+- else
+- c3cn_set_flag(c3cn, C3CN_CLOSE_NEEDED);
+- c3cn_put(c3cn);
++ wrlen = wr_len * 8;
+ }
+
+-
+-/*
+- * Local utility routines used to implement primary cxgb3 API operations.
+- * ======================================================================
+- */
+-
+-static u32 s3_send_rx_credits(struct s3_conn *, u32, u32, int);
+-static int act_open(struct s3_conn *, struct net_device *);
+-static void mk_act_open_req(struct s3_conn *, struct sk_buff *,
+- unsigned int, const struct l2t_entry *);
+-static void skb_entail(struct s3_conn *, struct sk_buff *, int);
+-
+ static inline void reset_wr_list(struct s3_conn *c3cn)
+ {
+ c3cn->wr_pending_head = NULL;
+@@ -532,7 +499,7 @@ static inline void enqueue_wr(struct s3_
+
+ /*
+ * We want to take an extra reference since both us and the driver
+- * need to free the packet before it's really freed. We know there's
++ * need to free the packet before it's really freed. We know there's
+ * just one user currently so we use atomic_set rather than skb_get
+ * to avoid the atomic op.
+ */
+@@ -545,34 +512,37 @@ static inline void enqueue_wr(struct s3_
+ c3cn->wr_pending_tail = skb;
+ }
+
+-/*
+- * The next two functions calculate the option 0 value for a connection.
+- */
+-static inline int compute_wscale(int win)
++static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
+ {
+- int wscale = 0;
+- while (wscale < 14 && (65535<<wscale) < win)
+- wscale++;
+- return wscale;
++ return c3cn->wr_pending_head;
+ }
+
+-static inline unsigned int calc_opt0h(struct s3_conn *c3cn)
++static inline void free_wr_skb(struct sk_buff *skb)
+ {
+- int wscale = compute_wscale(cxgb3_rcv_win);
+- return V_KEEP_ALIVE(1) |
+- F_TCAM_BYPASS |
+- V_WND_SCALE(wscale) |
+- V_MSS_IDX(c3cn->mss_idx);
++ kfree_skb(skb);
+ }
+
+-static inline unsigned int calc_opt0l(struct s3_conn *c3cn)
++static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
+ {
+- return V_ULP_MODE(ULP_MODE_ISCSI) |
+- V_RCV_BUFSIZ(cxgb3_rcv_win>>10);
++ struct sk_buff *skb = c3cn->wr_pending_head;
++
++ if (likely(skb)) {
++ /* Don't bother clearing the tail */
++ c3cn->wr_pending_head = (struct sk_buff *)skb->sp;
++ skb->sp = NULL;
++ }
++ return skb;
++}
++
++static void purge_wr_queue(struct s3_conn *c3cn)
++{
++ struct sk_buff *skb;
++ while ((skb = dequeue_wr(c3cn)) != NULL)
++ free_wr_skb(skb);
+ }
+
+-static inline void make_tx_data_wr(struct s3_conn *c3cn,
+- struct sk_buff *skb, int len)
++static inline void make_tx_data_wr(struct s3_conn *c3cn, struct sk_buff *skb,
++ int len)
+ {
+ struct tx_data_wr *req;
+
+@@ -591,250 +561,63 @@ static inline void make_tx_data_wr(struc
+ if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
+ req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
+ V_TX_CPU_IDX(c3cn->qset));
+-
+- /* Sendbuffer is in units of 32KB.
+- */
++ /* Sendbuffer is in units of 32KB. */
+ req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
+ c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
+ }
+ }
+
+ /**
+- * cxgb3_egress_dev - return the cxgb3 egress device
+- * @root_dev: the root device anchoring the search
+- * @c3cn: the connection used to determine egress port in bonding mode
+- * @context: in bonding mode, indicates a connection set up or failover
++ * c3cn_push_tx_frames -- start transmit
++ * @c3cn: the offloaded connection
++ * @req_completion: request wr_ack or not
+ *
+- * Return egress device or NULL if the egress device isn't one of our ports.
+- *
+- * Given a root network device it returns the physical egress device that is a
+- * descendant of the root device. The root device may be either a physical
+- * device, in which case it is the device returned, or a virtual device, such
+- * as a VLAN or bonding device. In case of a bonding device the search
+- * considers the decisions of the bonding device given its mode to locate the
+- * correct egress device.
++ * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
++ * connection's send queue and sends them on to T3. Must be called with the
++ * connection's lock held. Returns the amount of send buffer space that was
++ * freed as a result of sending queued data to T3.
+ */
+-static struct net_device *cxgb3_egress_dev(struct net_device *root_dev,
+- struct s3_conn *c3cn,
+- int context)
+-{
+- while (root_dev) {
+- if (root_dev->priv_flags & IFF_802_1Q_VLAN)
+- root_dev = vlan_dev_real_dev(root_dev);
+- else if (is_cxgb3_dev(root_dev))
+- return root_dev;
+- else
+- return NULL;
+- }
+- return NULL;
+-}
+-
+-static struct rtable *find_route(__be32 saddr, __be32 daddr,
+- __be16 sport, __be16 dport)
++static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb)
+ {
+- struct rtable *rt;
+- struct flowi fl = {
+- .oif = 0,
+- .nl_u = {
+- .ip4_u = {
+- .daddr = daddr,
+- .saddr = saddr,
+- .tos = 0 } },
+- .proto = IPPROTO_TCP,
+- .uli_u = {
+- .ports = {
+- .sport = sport,
+- .dport = dport } } };
+-
+- if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
+- return NULL;
+- return rt;
++ kfree_skb(skb);
+ }
+
+-int cxgb3i_c3cn_connect(struct s3_conn *c3cn, struct sockaddr_in *usin)
++static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion)
+ {
+- struct rtable *rt;
+- struct net_device *dev;
+- struct cxgb3i_sdev_data *cdata;
++ int total_size = 0;
++ struct sk_buff *skb;
+ struct t3cdev *cdev;
+- __be32 sipv4;
+- int err;
++ struct cxgb3i_sdev_data *cdata;
+
+- if (usin->sin_family != AF_INET)
+- return -EAFNOSUPPORT;
++ if (unlikely(c3cn->state == C3CN_STATE_CONNECTING ||
++ c3cn->state == C3CN_STATE_CLOSE_WAIT_1 ||
++ c3cn->state == C3CN_STATE_ABORTING)) {
++ c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
++ c3cn, c3cn->state);
++ return 0;
++ }
+
+- /* get a source port if one hasn't been provided */
+- err = c3cn_get_port(c3cn);
+- if (err)
+- return err;
++ cdev = c3cn->cdev;
++ cdata = CXGB3_SDEV_DATA(cdev);
+
+- c3cn_conn_debug("c3cn 0x%p get port %u.\n",
+- c3cn, ntohs(c3cn->saddr.sin_port));
++ while (c3cn->wr_avail
++ && (skb = skb_peek(&c3cn->write_queue)) != NULL) {
++ int len = skb->len; /* length before skb_push */
++ int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
++ int wrs_needed = skb_wrs[frags];
+
+- c3cn->daddr.sin_port = usin->sin_port;
+- c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr;
++ if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
++ wrs_needed = 1;
+
+- rt = find_route(c3cn->saddr.sin_addr.s_addr,
+- c3cn->daddr.sin_addr.s_addr,
+- c3cn->saddr.sin_port,
+- c3cn->daddr.sin_port);
+- if (rt == NULL) {
+- c3cn_conn_debug("NO route to 0x%x, port %u.\n",
+- c3cn->daddr.sin_addr.s_addr,
+- ntohs(c3cn->daddr.sin_port));
+- return -ENETUNREACH;
+- }
++ WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
+
+- if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
+- c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n",
+- c3cn->daddr.sin_addr.s_addr,
+- ntohs(c3cn->daddr.sin_port));
+- ip_rt_put(rt);
+- return -ENETUNREACH;
+- }
+-
+- if (!c3cn->saddr.sin_addr.s_addr)
+- c3cn->saddr.sin_addr.s_addr = rt->rt_src;
+-
+- /* now commit destination to connection */
+- c3cn->dst_cache = &rt->u.dst;
+-
+- /* try to establish an offloaded connection */
+- dev = cxgb3_egress_dev(c3cn->dst_cache->dev, c3cn, 0);
+- if (dev == NULL) {
+- c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn);
+- return -ENETUNREACH;
+- }
+- cdata = NDEV2CDATA(dev);
+- cdev = cdata->cdev;
+-
+- sipv4 = cxgb3i_get_private_ipv4addr(dev);
+- if (!sipv4) {
+- c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn);
+- sipv4 = c3cn->saddr.sin_addr.s_addr;
+- cxgb3i_set_private_ipv4addr(dev, sipv4);
+- } else
+- c3cn->saddr.sin_addr.s_addr = sipv4;
+-
+- c3cn_conn_debug("c3cn 0x%p, %u.%u.%u.%u,%u-%u.%u.%u.%u,%u SYN_SENT.\n",
+- c3cn, NIPQUAD(c3cn->saddr.sin_addr.s_addr),
+- ntohs(c3cn->saddr.sin_port),
+- NIPQUAD(c3cn->daddr.sin_addr.s_addr),
+- ntohs(c3cn->daddr.sin_port));
+-
+- c3cn_set_state(c3cn, C3CN_STATE_SYN_SENT);
+-
+- if (!act_open(c3cn, dev))
+- return 0;
+-
+- /*
+- * If we get here, we don't have an offload connection so simply
+- * return a failure.
+- */
+- err = -ENOTSUPP;
+-
+- /*
+- * This trashes the connection and releases the local port,
+- * if necessary.
+- */
+- c3cn_conn_debug("c3cn 0x%p -> CLOSE.\n", c3cn);
+- c3cn_set_state(c3cn, C3CN_STATE_CLOSE);
+- ip_rt_put(rt);
+- c3cn_put_port(c3cn);
+- c3cn->daddr.sin_port = 0;
+- return err;
+-}
+-
+-/*
+- * Set of states for which we should return RX credits.
+- */
+-#define CREDIT_RETURN_STATE (C3CN_STATE_ESTABLISHED)
+-
+-/*
+- * Called after some received data has been read. It returns RX credits
+- * to the HW for the amount of data processed.
+- */
+-void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied)
+-{
+- struct t3cdev *cdev;
+- int must_send;
+- u32 credits, dack = 0;
+-
+- if (!c3cn_in_state(c3cn, CREDIT_RETURN_STATE))
+- return;
+-
+- credits = c3cn->copied_seq - c3cn->rcv_wup;
+- if (unlikely(!credits))
+- return;
+-
+- cdev = c3cn->cdev;
+-
+- if (unlikely(cxgb3_rx_credit_thres == 0))
+- return;
+-
+- dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
+-
+- /*
+- * For coalescing to work effectively ensure the receive window has
+- * at least 16KB left.
+- */
+- must_send = credits + 16384 >= cxgb3_rcv_win;
+-
+- if (must_send || credits >= cxgb3_rx_credit_thres)
+- c3cn->rcv_wup += s3_send_rx_credits(c3cn, credits, dack,
+- must_send);
+-}
+-
+-/*
+- * Generic ARP failure handler that discards the buffer.
+- */
+-static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb)
+-{
+- kfree_skb(skb);
+-}
+-
+-/*
+- * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
+- * connection's send queue and sends them on to T3. Must be called with the
+- * connection's lock held. Returns the amount of send buffer space that was
+- * freed as a result of sending queued data to T3.
+- */
+-static int s3_push_frames(struct s3_conn *c3cn, int req_completion)
+-{
+- int total_size = 0;
+- struct sk_buff *skb;
+- struct t3cdev *cdev;
+- struct cxgb3i_sdev_data *cdata;
+-
+- if (unlikely(c3cn_in_state(c3cn,
+- C3CN_STATE_SYN_SENT | C3CN_STATE_CLOSE)))
+- return 0;
+-
+- /*
+- * We shouldn't really be called at all after an abort but check just
+- * in case.
+- */
+- if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN)))
+- return 0;
+-
+- cdev = c3cn->cdev;
+- cdata = CXGB3_SDEV_DATA(cdev);
+-
+- while (c3cn->wr_avail
+- && (skb = skb_peek(&c3cn->write_queue)) != NULL
+- && !c3cn_flag(c3cn, C3CN_TX_WAIT_IDLE)) {
+-
+- int len = skb->len; /* length before skb_push */
+- int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
+- int wrs_needed = skb_wrs[frags];
+-
+- if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
+- wrs_needed = 1;
+-
+- WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
+-
+- if (c3cn->wr_avail < wrs_needed)
++ if (c3cn->wr_avail < wrs_needed) {
++ c3cn_tx_debug("c3cn 0x%p, skb len %u/%u, frag %u, "
++ "wr %d < %u.\n",
++ c3cn, skb->len, skb->data_len, frags,
++ wrs_needed, c3cn->wr_avail);
+ break;
++ }
+
+ __skb_unlink(skb, &c3cn->write_queue);
+ skb->priority = CPL_PRIORITY_DATA;
+@@ -857,8 +640,7 @@ static int s3_push_frames(struct s3_conn
+ c3cn->wr_unacked = 0;
+ }
+ CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR;
+- } else if (skb->data[0] == FW_WROPCODE_OFLD_CLOSE_CON)
+- c3cn_set_flag(c3cn, C3CN_CLOSE_CON_REQUESTED);
++ }
+
+ total_size += skb->truesize;
+ set_arp_failure_handler(skb, arp_failure_discard);
+@@ -868,515 +650,386 @@ static int s3_push_frames(struct s3_conn
+ }
+
+ /*
+- * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant
+- * and send it along.
++ * process_cpl_msg: -> host
++ * Top-level CPL message processing used by most CPL messages that
++ * pertain to connections.
+ */
+-static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb)
++static inline void process_cpl_msg(void (*fn)(struct s3_conn *,
++ struct sk_buff *),
++ struct s3_conn *c3cn,
++ struct sk_buff *skb)
+ {
+- struct cpl_abort_req *req = cplhdr(skb);
+-
+- c3cn_conn_debug("tdev 0x%p.\n", cdev);
+-
+- req->cmd = CPL_ABORT_NO_RST;
+- cxgb3_ofld_send(cdev, skb);
++ spin_lock_bh(&c3cn->lock);
++ fn(c3cn, skb);
++ spin_unlock_bh(&c3cn->lock);
+ }
+
+ /*
+- * Send an ABORT_REQ message. Cannot fail. This routine makes sure we do
+- * not send multiple ABORT_REQs for the same connection and also that we do
+- * not try to send a message after the connection has closed. Returns 1 if
+- * an ABORT_REQ wasn't generated after all, 0 otherwise.
++ * process_cpl_msg_ref: -> host
++ * Similar to process_cpl_msg() but takes an extra connection reference around
++ * the call to the handler. Should be used if the handler may drop a
++ * connection reference.
+ */
+-static int s3_send_reset(struct s3_conn *c3cn, int mode,
+- struct sk_buff *skb)
++static inline void process_cpl_msg_ref(void (*fn) (struct s3_conn *,
++ struct sk_buff *),
++ struct s3_conn *c3cn,
++ struct sk_buff *skb)
+ {
+- struct cpl_abort_req *req;
+- unsigned int tid = c3cn->tid;
++ c3cn_hold(c3cn);
++ process_cpl_msg(fn, c3cn, skb);
++ c3cn_put(c3cn);
++}
+
+- if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN) || !c3cn->cdev)) {
+- if (skb)
+- __kfree_skb(skb);
+- return 1;
+- }
++/*
++ * Process a CPL_ACT_ESTABLISH message: -> host
++ * Updates connection state from an active establish CPL message. Runs with
++ * the connection lock held.
++ */
+
+- c3cn_conn_debug("c3cn 0x%p, mode %d, flag ABORT_RPL + ABORT_SHUT.\n",
+- c3cn, mode);
++static inline void s3_free_atid(struct t3cdev *cdev, unsigned int tid)
++{
++ struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid);
++ if (c3cn)
++ c3cn_put(c3cn);
++}
+
+- c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING);
+- c3cn_set_flag(c3cn, C3CN_ABORT_SHUTDOWN);
++static void c3cn_established(struct s3_conn *c3cn, u32 snd_isn,
++ unsigned int opt)
++{
++ c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
+
+- /* Purge the send queue so we don't send anything after an abort. */
+- s3_purge_write_queue(c3cn);
++ c3cn->write_seq = c3cn->snd_nxt = c3cn->snd_una = snd_isn;
+
+- if (!skb)
+- skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
+- skb->priority = CPL_PRIORITY_DATA;
+- set_arp_failure_handler(skb, abort_arp_failure);
++ /*
++ * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
++ * pass through opt0.
++ */
++ if (cxgb3_rcv_win > (M_RCV_BUFSIZ << 10))
++ c3cn->rcv_wup -= cxgb3_rcv_win - (M_RCV_BUFSIZ << 10);
+
+- req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
+- req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
+- req->wr.wr_lo = htonl(V_WR_TID(tid));
+- OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
+- req->rsvd0 = htonl(c3cn->snd_nxt);
+- req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT);
+- req->cmd = mode;
++ dst_confirm(c3cn->dst_cache);
+
+- l2t_send(c3cn->cdev, skb, c3cn->l2t);
+- return 0;
++ smp_mb();
++
++ c3cn_set_state(c3cn, C3CN_STATE_ESTABLISHED);
+ }
+
+-/*
+- * Add a list of skbs to a connection send queue. This interface is intended
+- * for use by in-kernel ULPs. The skbs must comply with the max size limit of
+- * the device and have a headroom of at least TX_HEADER_LEN bytes.
+- */
+-int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb, int flags)
++static void process_act_establish(struct s3_conn *c3cn, struct sk_buff *skb)
+ {
+- struct sk_buff *next;
+- int err, copied = 0;
+-
+- spin_lock_bh(&c3cn->lock);
++ struct cpl_act_establish *req = cplhdr(skb);
++ u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
+
+- if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED)) {
+- err = -EAGAIN;
+- goto out_err;
+- }
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
+
+- err = -EPIPE;
+- if (c3cn->err || (c3cn->shutdown & C3CN_SEND_SHUTDOWN))
+- goto out_err;
++ if (unlikely(c3cn->state != C3CN_STATE_CONNECTING))
++ cxgb3i_log_error("TID %u expected SYN_SENT, got EST., s %u\n",
++ c3cn->tid, c3cn->state);
+
+- while (skb) {
+- int frags = skb_shinfo(skb)->nr_frags +
+- (skb->len != skb->data_len);
++ c3cn->copied_seq = c3cn->rcv_wup = c3cn->rcv_nxt = rcv_isn;
++ c3cn_established(c3cn, ntohl(req->snd_isn), ntohs(req->tcp_opt));
+
+- if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
+- c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
+- err = -EINVAL;
+- goto out_err;
+- }
++ __kfree_skb(skb);
+
+- if (frags >= SKB_WR_LIST_SIZE) {
+- cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
+- c3cn, skb_shinfo(skb)->nr_frags,
+- skb->len, skb->data_len);
+- err = -EINVAL;
+- goto out_err;
+- }
++ if (unlikely(c3cn_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED)))
++ /* upper layer has requested closing */
++ send_abort_req(c3cn);
++ else if (c3cn_push_tx_frames(c3cn, 1))
++ cxgb3i_conn_tx_open(c3cn);
++}
+
+- next = skb->next;
+- skb->next = NULL;
+- skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
+- copied += skb->len;
+- c3cn->write_seq += skb->len + ulp_extra_len(skb);
+- skb = next;
+- }
+-done:
+- if (likely(skb_queue_len(&c3cn->write_queue)))
+- s3_push_frames(c3cn, 1);
+- spin_unlock_bh(&c3cn->lock);
+- return copied;
++static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
++ void *ctx)
++{
++ struct cpl_act_establish *req = cplhdr(skb);
++ unsigned int tid = GET_TID(req);
++ unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
++ struct s3_conn *c3cn = ctx;
++ struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
+
+-out_err:
+- if (copied == 0 && err == -EPIPE)
+- copied = c3cn->err ? c3cn->err : -EPIPE;
+- goto done;
+-}
++ c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
++ tid, c3cn, c3cn->state, c3cn->flags);
+
+-/*
+- * Low-level utility routines for primary API functions.
+- * =====================================================
+- */
+-/* routines to implement CPL message processing */
+-static void c3cn_act_establish(struct s3_conn *, struct sk_buff *);
+-static void active_open_failed(struct s3_conn *, struct sk_buff *);
+-static void wr_ack(struct s3_conn *, struct sk_buff *);
+-static void do_peer_fin(struct s3_conn *, struct sk_buff *);
+-static void process_abort_req(struct s3_conn *, struct sk_buff *);
+-static void process_abort_rpl(struct s3_conn *, struct sk_buff *);
+-static void process_close_con_rpl(struct s3_conn *, struct sk_buff *);
+-static void process_rx_iscsi_hdr(struct s3_conn *, struct sk_buff *);
++ c3cn->tid = tid;
++ c3cn_hold(c3cn);
++ cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid);
++ s3_free_atid(cdev, atid);
+
+-static struct sk_buff *__get_cpl_reply_skb(struct sk_buff *, size_t, gfp_t);
++ c3cn->qset = G_QNUM(ntohl(skb->csum));
+
+-static void fail_act_open(struct s3_conn *, int);
+-static void init_offload_conn(struct s3_conn *, struct t3cdev *,
+- struct dst_entry *);
++ process_cpl_msg(process_act_establish, c3cn, skb);
++ return 0;
++}
+
+ /*
+- * Insert a connection into the TID table and take an extra reference.
++ * Process a CPL_ACT_OPEN_RPL message: -> host
++ * Handle active open failures.
+ */
+-static inline void c3cn_insert_tid(struct cxgb3i_sdev_data *cdata,
+- struct s3_conn *c3cn,
+- unsigned int tid)
++static int act_open_rpl_status_to_errno(int status)
+ {
+- c3cn_hold(c3cn);
+- cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid);
+-}
+-
+-static inline void free_atid(struct t3cdev *cdev, unsigned int tid)
+-{
+- struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid);
+- if (c3cn)
+- c3cn_put(c3cn);
+-}
+-
+-/*
+- * This function is intended for allocations of small control messages.
+- * Such messages go as immediate data and usually the pakets are freed
+- * immediately. We maintain a cache of one small sk_buff and use it whenever
+- * it is available (has a user count of 1). Otherwise we get a fresh buffer.
+- */
+-#define CTRL_SKB_LEN 120
+-
+-static struct sk_buff *alloc_ctrl_skb(const struct s3_conn *c3cn,
+- int len)
+-{
+- struct sk_buff *skb = c3cn->ctrl_skb_cache;
+-
+- if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
+- __skb_trim(skb, 0);
+- atomic_set(&skb->users, 2);
+- } else if (likely(!in_atomic()))
+- skb = alloc_skb(len, GFP_ATOMIC | __GFP_NOFAIL);
+- else
+- skb = alloc_skb(len, GFP_ATOMIC);
+- return skb;
++ switch (status) {
++ case CPL_ERR_CONN_RESET:
++ return ECONNREFUSED;
++ case CPL_ERR_ARP_MISS:
++ return EHOSTUNREACH;
++ case CPL_ERR_CONN_TIMEDOUT:
++ return ETIMEDOUT;
++ case CPL_ERR_TCAM_FULL:
++ return ENOMEM;
++ case CPL_ERR_CONN_EXIST:
++ cxgb3i_log_error("ACTIVE_OPEN_RPL: 4-tuple in use\n");
++ return EADDRINUSE;
++ default:
++ return EIO;
++ }
+ }
+
+-/*
+- * Handle an ARP failure for an active open.
+- */
+-static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
++static void act_open_retry_timer(unsigned long data)
+ {
+- struct s3_conn *c3cn = (struct s3_conn *)skb->sk;
++ struct sk_buff *skb;
++ struct s3_conn *c3cn = (struct s3_conn *)data;
+
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
++ c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
+
+- c3cn_hold(c3cn);
+- spin_lock(&c3cn->lock);
+- if (c3cn->state == C3CN_STATE_SYN_SENT) {
+- fail_act_open(c3cn, EHOSTUNREACH);
+- __kfree_skb(skb);
++ spin_lock_bh(&c3cn->lock);
++ skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_ATOMIC);
++ if (!skb)
++ fail_act_open(c3cn, ENOMEM);
++ else {
++ skb->sk = (struct sock *)c3cn;
++ set_arp_failure_handler(skb, act_open_req_arp_failure);
++ make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
++ l2t_send(c3cn->cdev, skb, c3cn->l2t);
+ }
+- spin_unlock(&c3cn->lock);
++ spin_unlock_bh(&c3cn->lock);
+ c3cn_put(c3cn);
+ }
+
+-/*
+- * Send an active open request.
+- */
+-static int act_open(struct s3_conn *c3cn, struct net_device *dev)
++static void process_act_open_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
+ {
+- struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev);
+- struct t3cdev *cdev = cdata->cdev;
+- struct dst_entry *dst = c3cn->dst_cache;
+- struct sk_buff *skb;
++ struct cpl_act_open_rpl *rpl = cplhdr(skb);
+
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
+ c3cn, c3cn->state, c3cn->flags);
+- /*
+- * Initialize connection data. Note that the flags and ULP mode are
+- * initialized higher up ...
+- */
+- c3cn->dev = dev;
+- c3cn->cdev = cdev;
+- c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn);
+- if (c3cn->tid < 0)
+- goto out_err;
+-
+- c3cn->qset = 0;
+- c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev);
+- if (!c3cn->l2t)
+- goto free_tid;
+
+- skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL);
+- if (!skb)
+- goto free_l2t;
++ if (rpl->status == CPL_ERR_CONN_EXIST &&
++ c3cn->retry_timer.function != act_open_retry_timer) {
++ c3cn->retry_timer.function = act_open_retry_timer;
++ if (!mod_timer(&c3cn->retry_timer, jiffies + HZ / 2))
++ c3cn_hold(c3cn);
++ } else
++ fail_act_open(c3cn, act_open_rpl_status_to_errno(rpl->status));
++ __kfree_skb(skb);
++}
+
+- skb->sk = (struct sock *)c3cn;
+- set_arp_failure_handler(skb, act_open_req_arp_failure);
++static int do_act_open_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
++{
++ struct s3_conn *c3cn = ctx;
++ struct cpl_act_open_rpl *rpl = cplhdr(skb);
+
+- c3cn_hold(c3cn);
++ c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
++ rpl->status, c3cn, c3cn->state, c3cn->flags);
+
+- init_offload_conn(c3cn, cdev, dst);
+- c3cn->err = 0;
+- c3cn_reset_flag(c3cn, C3CN_DONE);
++ if (rpl->status != CPL_ERR_TCAM_FULL &&
++ rpl->status != CPL_ERR_CONN_EXIST &&
++ rpl->status != CPL_ERR_ARP_MISS)
++ cxgb3_queue_tid_release(cdev, GET_TID(rpl));
+
+- mk_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
+- l2t_send(cdev, skb, c3cn->l2t);
++ process_cpl_msg_ref(process_act_open_rpl, c3cn, skb);
+ return 0;
+-
+-free_l2t:
+- l2t_release(L2DATA(cdev), c3cn->l2t);
+-free_tid:
+- free_atid(cdev, c3cn->tid);
+- c3cn->tid = 0;
+-out_err:
+- return -1;
+ }
+
+ /*
+- * Close a connection by sending a CPL_CLOSE_CON_REQ message. Cannot fail
+- * under any circumstances. We take the easy way out and always queue the
+- * message to the write_queue. We can optimize the case where the queue is
+- * already empty though the optimization is probably not worth it.
++ * Process PEER_CLOSE CPL messages: -> host
++ * Handle peer FIN.
+ */
+-static void mk_close_req(struct s3_conn *c3cn)
++static void process_peer_close(struct s3_conn *c3cn, struct sk_buff *skb)
+ {
+- struct sk_buff *skb;
+- struct cpl_close_con_req *req;
+- unsigned int tid = c3cn->tid;
+-
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
+ c3cn, c3cn->state, c3cn->flags);
+
+- skb = alloc_skb(sizeof(struct cpl_close_con_req),
+- GFP_KERNEL | __GFP_NOFAIL);
+- req = (struct cpl_close_con_req *)__skb_put(skb, sizeof(*req));
+- req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
+- req->wr.wr_lo = htonl(V_WR_TID(tid));
+- OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
+- req->rsvd = htonl(c3cn->write_seq);
++ if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
++ goto out;
+
+- skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND);
+- if (c3cn->state != C3CN_STATE_SYN_SENT)
+- s3_push_frames(c3cn, 1);
++ switch (c3cn->state) {
++ case C3CN_STATE_ESTABLISHED:
++ c3cn_set_state(c3cn, C3CN_STATE_PASSIVE_CLOSE);
++ break;
++ case C3CN_STATE_ACTIVE_CLOSE:
++ c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
++ break;
++ case C3CN_STATE_CLOSE_WAIT_1:
++ c3cn_closed(c3cn);
++ break;
++ case C3CN_STATE_ABORTING:
++ break;
++ default:
++ cxgb3i_log_error("%s: peer close, TID %u in bad state %u\n",
++ c3cn->cdev->name, c3cn->tid, c3cn->state);
++ }
++
++ cxgb3i_conn_closing(c3cn);
++out:
++ __kfree_skb(skb);
+ }
+
+-static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
+- int flags)
++static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+ {
+- CXGB3_SKB_CB(skb)->seq = c3cn->write_seq;
+- CXGB3_SKB_CB(skb)->flags = flags;
+- __skb_queue_tail(&c3cn->write_queue, skb);
++ struct s3_conn *c3cn = ctx;
++
++ c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++ process_cpl_msg_ref(process_peer_close, c3cn, skb);
++ return 0;
+ }
+
+ /*
+- * Send RX credits through an RX_DATA_ACK CPL message. If nofail is 0 we are
+- * permitted to return without sending the message in case we cannot allocate
+- * an sk_buff. Returns the number of credits sent.
++ * Process CLOSE_CONN_RPL CPL message: -> host
++ * Process a peer ACK to our FIN.
+ */
+-static u32 s3_send_rx_credits(struct s3_conn *c3cn, u32 credits, u32 dack,
+- int nofail)
++static void process_close_con_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
+ {
+- struct sk_buff *skb;
+- struct cpl_rx_data_ack *req;
++ struct cpl_close_con_rpl *rpl = cplhdr(skb);
+
+- skb = (nofail ? alloc_ctrl_skb(c3cn, sizeof(*req))
+- : alloc_skb(sizeof(*req), GFP_ATOMIC));
+- if (!skb)
+- return 0;
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
+
+- req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
+- req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+- OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, c3cn->tid));
+- req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
+- skb->priority = CPL_PRIORITY_ACK;
+- cxgb3_ofld_send(c3cn->cdev, skb);
+- return credits;
+-}
++ c3cn->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
+
+-static void mk_act_open_req(struct s3_conn *c3cn, struct sk_buff *skb,
+- unsigned int atid, const struct l2t_entry *e)
+-{
+- struct cpl_act_open_req *req;
++ if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
++ goto out;
+
+- c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn, atid);
++ switch (c3cn->state) {
++ case C3CN_STATE_ACTIVE_CLOSE:
++ c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_1);
++ break;
++ case C3CN_STATE_CLOSE_WAIT_1:
++ case C3CN_STATE_CLOSE_WAIT_2:
++ c3cn_closed(c3cn);
++ break;
++ case C3CN_STATE_ABORTING:
++ break;
++ default:
++ cxgb3i_log_error("%s: close_rpl, TID %u in bad state %u\n",
++ c3cn->cdev->name, c3cn->tid, c3cn->state);
++ }
+
+- skb->priority = CPL_PRIORITY_SETUP;
+- req = (struct cpl_act_open_req *)__skb_put(skb, sizeof(*req));
+- req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+- OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
+- req->local_port = c3cn->saddr.sin_port;
+- req->peer_port = c3cn->daddr.sin_port;
+- req->local_ip = c3cn->saddr.sin_addr.s_addr;
+- req->peer_ip = c3cn->daddr.sin_addr.s_addr;
+- req->opt0h = htonl(calc_opt0h(c3cn) | V_L2T_IDX(e->idx) |
+- V_TX_CHANNEL(e->smt_idx));
+- req->opt0l = htonl(calc_opt0l(c3cn));
+- req->params = 0;
++out:
++ kfree_skb(skb);
+ }
+
+-/*
+- * Definitions and declarations for CPL handler functions.
+- * =======================================================
+- */
+-
+-/*
+- * Similar to process_cpl_msg() but takes an extra connection reference around
+- * the call to the handler. Should be used if the handler may drop a
+- * connection reference.
+- */
+-static inline void process_cpl_msg_ref(void (*fn) (struct s3_conn *,
+- struct sk_buff *),
+- struct s3_conn *c3cn,
+- struct sk_buff *skb)
++static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
++ void *ctx)
+ {
+- c3cn_hold(c3cn);
+- process_cpl_msg(fn, c3cn, skb);
+- c3cn_put(c3cn);
+-}
++ struct s3_conn *c3cn = ctx;
+
+-/*
+- * Return whether a failed active open has allocated a TID
+- */
+-static inline int act_open_has_tid(int status)
+-{
+- return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST &&
+- status != CPL_ERR_ARP_MISS;
++ c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ process_cpl_msg_ref(process_close_con_rpl, c3cn, skb);
++ return 0;
+ }
+
+ /*
+- * Returns true if a connection cannot accept new Rx data.
++ * Process ABORT_REQ_RSS CPL message: -> host
++ * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
++ * request except that we need to reply to it.
+ */
+-static inline int c3cn_no_receive(const struct s3_conn *c3cn)
++
++static int abort_status_to_errno(struct s3_conn *c3cn, int abort_reason,
++ int *need_rst)
+ {
+- return c3cn->shutdown & C3CN_RCV_SHUTDOWN;
++ switch (abort_reason) {
++ case CPL_ERR_BAD_SYN: /* fall through */
++ case CPL_ERR_CONN_RESET:
++ return c3cn->state > C3CN_STATE_ESTABLISHED ?
++ EPIPE : ECONNRESET;
++ case CPL_ERR_XMIT_TIMEDOUT:
++ case CPL_ERR_PERSIST_TIMEDOUT:
++ case CPL_ERR_FINWAIT2_TIMEDOUT:
++ case CPL_ERR_KEEPALIVE_TIMEDOUT:
++ return ETIMEDOUT;
++ default:
++ return EIO;
++ }
+ }
+
+-/*
+- * A helper function that aborts a connection and increments the given MIB
+- * counter. The supplied skb is used to generate the ABORT_REQ message if
+- * possible. Must be called with softirqs disabled.
+- */
+-static inline void abort_conn(struct s3_conn *c3cn,
+- struct sk_buff *skb)
++static void process_abort_req(struct s3_conn *c3cn, struct sk_buff *skb)
+ {
+- struct sk_buff *abort_skb;
++ int rst_status = CPL_ABORT_NO_RST;
++ const struct cpl_abort_req_rss *req = cplhdr(skb);
+
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
+ c3cn, c3cn->state, c3cn->flags);
+
+- abort_skb = __get_cpl_reply_skb(skb, sizeof(struct cpl_abort_req),
+- GFP_ATOMIC);
+- if (abort_skb)
+- s3_send_reset(c3cn, CPL_ABORT_SEND_RST, abort_skb);
+-}
++ if (!c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD)) {
++ c3cn_set_flag(c3cn, C3CN_ABORT_REQ_RCVD);
++ c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
++ __kfree_skb(skb);
++ return;
++ }
+
+-/*
+- * Returns whether an ABORT_REQ_RSS message is a negative advice.
+- */
+-static inline int is_neg_adv_abort(unsigned int status)
+-{
+- return status == CPL_ERR_RTX_NEG_ADVICE ||
+- status == CPL_ERR_PERSIST_NEG_ADVICE;
+-}
++ c3cn_clear_flag(c3cn, C3CN_ABORT_REQ_RCVD);
++ send_abort_rpl(c3cn, rst_status);
+
+-/*
+- * CPL handler functions.
+- * ======================
+- */
++ if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
++ c3cn->err =
++ abort_status_to_errno(c3cn, req->status, &rst_status);
++ c3cn_closed(c3cn);
++ }
++}
+
+-/*
+- * Process a CPL_ACT_ESTABLISH message.
+- */
+-static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
+- void *ctx)
++static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+ {
+- struct cpl_act_establish *req = cplhdr(skb);
+- unsigned int tid = GET_TID(req);
+- unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
++ const struct cpl_abort_req_rss *req = cplhdr(skb);
+ struct s3_conn *c3cn = ctx;
+- struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
+
+- c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
+- tid, c3cn, c3cn->state, c3cn->flags);
+- /*
+- * It's OK if the TID is currently in use, the owning connection may
+- * have backlogged its last CPL message(s). Just take it away.
+- */
+- c3cn->tid = tid;
+- c3cn_insert_tid(cdata, c3cn, tid);
+- free_atid(cdev, atid);
++ c3cn_conn_debug("rcv, c3cn 0x%p, s 0x%x, f 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
+
+- c3cn->qset = G_QNUM(ntohl(skb->csum));
++ if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
++ req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
++ __kfree_skb(skb);
++ return 0;
++ }
+
+- process_cpl_msg(c3cn_act_establish, c3cn, skb);
++ process_cpl_msg_ref(process_abort_req, c3cn, skb);
+ return 0;
+ }
+
+ /*
+- * Process an ACT_OPEN_RPL CPL message.
++ * Process ABORT_RPL_RSS CPL message: -> host
++ * Process abort replies. We only process these messages if we anticipate
++ * them as the coordination between SW and HW in this area is somewhat lacking
++ * and sometimes we get ABORT_RPLs after we are done with the connection that
++ * originated the ABORT_REQ.
+ */
+-static int do_act_open_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
++static void process_abort_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
+ {
+- struct s3_conn *c3cn = ctx;
+- struct cpl_act_open_rpl *rpl = cplhdr(skb);
+-
+- c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
+- rpl->status, c3cn, c3cn->state, c3cn->flags);
+-
+- if (act_open_has_tid(rpl->status))
+- cxgb3_queue_tid_release(cdev, GET_TID(rpl));
+-
+- process_cpl_msg_ref(active_open_failed, c3cn, skb);
+- return 0;
+-}
+-
+-/*
+- * Handler RX_ISCSI_HDR CPL messages.
+- */
+-static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
+-{
+- struct s3_conn *c3cn = ctx;
+- process_cpl_msg(process_rx_iscsi_hdr, c3cn, skb);
+- return 0;
+-}
+-
+-/*
+- * Handler for TX_DATA_ACK CPL messages.
+- */
+-static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+-{
+- struct s3_conn *c3cn = ctx;
+-
+- process_cpl_msg(wr_ack, c3cn, skb);
+- return 0;
+-}
+-
+-/*
+- * Handler for PEER_CLOSE CPL messages.
+- */
+-static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+-{
+- struct s3_conn *c3cn = ctx;
+-
+- c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
+- process_cpl_msg_ref(do_peer_fin, c3cn, skb);
+- return 0;
+-}
+-
+-/*
+- * Handle an ABORT_REQ_RSS CPL message.
+- */
+-static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+-{
+- const struct cpl_abort_req_rss *req = cplhdr(skb);
+- struct s3_conn *c3cn = ctx;
+-
+- c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
+ c3cn, c3cn->state, c3cn->flags);
+
+- if (is_neg_adv_abort(req->status)) {
+- __kfree_skb(skb);
+- return 0;
++ if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
++ if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_RCVD))
++ c3cn_set_flag(c3cn, C3CN_ABORT_RPL_RCVD);
++ else {
++ c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_RCVD);
++ c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_PENDING);
++ if (c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD))
++ cxgb3i_log_error("%s tid %u, ABORT_RPL_RSS\n",
++ c3cn->cdev->name, c3cn->tid);
++ c3cn_closed(c3cn);
++ }
+ }
+-
+- process_cpl_msg_ref(process_abort_req, c3cn, skb);
+- return 0;
++ __kfree_skb(skb);
+ }
+
+-/*
+- * Handle an ABORT_RPL_RSS CPL message.
+- */
+ static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+ {
+ struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
+ struct s3_conn *c3cn = ctx;
+
+- c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
++ c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, 0x%lx.\n",
+ rpl->status, c3cn, c3cn ? c3cn->state : 0,
+ c3cn ? c3cn->flags : 0UL);
+
+@@ -1394,7 +1047,7 @@ static int do_abort_rpl(struct t3cdev *c
+ * abort races with ABORT_REQ_RSS, the latter frees the connection
+ * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
+ * but FW turns the ABORT_REQ into a regular one and so we get
+- * ABORT_RPL_RSS with status 0 and no connection. Only on T3A.
++ * ABORT_RPL_RSS with status 0 and no connection.
+ */
+ if (!c3cn)
+ goto discard;
+@@ -1408,144 +1061,11 @@ discard:
+ }
+
+ /*
+- * Handler for CLOSE_CON_RPL CPL messages.
+- */
+-static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
+- void *ctx)
+-{
+- struct s3_conn *c3cn = ctx;
+-
+- c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
+-
+- process_cpl_msg_ref(process_close_con_rpl, c3cn, skb);
+- return 0;
+-}
+-
+-/*
+- * Definitions and declarations for CPL message processing.
+- * ========================================================
+- */
+-
+-static void make_established(struct s3_conn *, u32, unsigned int);
+-static void act_open_retry_timer(unsigned long);
+-static void mk_act_open_req(struct s3_conn *, struct sk_buff *,
+- unsigned int, const struct l2t_entry *);
+-static int act_open_rpl_status_to_errno(int);
+-static void handle_excess_rx(struct s3_conn *, struct sk_buff *);
+-static int abort_status_to_errno(struct s3_conn *, int, int *);
+-static void send_abort_rpl(struct sk_buff *, struct t3cdev *, int);
+-static struct sk_buff *get_cpl_reply_skb(struct sk_buff *, size_t, gfp_t);
+-
+-/*
+- * Dequeue and return the first unacknowledged's WR on a connections's pending
+- * list.
+- */
+-static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
+-{
+- struct sk_buff *skb = c3cn->wr_pending_head;
+-
+- if (likely(skb)) {
+- /* Don't bother clearing the tail */
+- c3cn->wr_pending_head = (struct sk_buff *)skb->sp;
+- skb->sp = NULL;
+- }
+- return skb;
+-}
+-
+-/*
+- * Return the first pending WR without removing it from the list.
+- */
+-static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
+-{
+- return c3cn->wr_pending_head;
+-}
+-
+-static inline void free_wr_skb(struct sk_buff *skb)
+-{
+- kfree_skb(skb);
+-}
+-
+-static void purge_wr_queue(struct s3_conn *c3cn)
+-{
+- struct sk_buff *skb;
+- while ((skb = dequeue_wr(c3cn)) != NULL)
+- free_wr_skb(skb);
+-}
+-
+-static inline void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
+- int cmd)
+-{
+- struct cpl_abort_rpl *rpl = cplhdr(skb);
+-
+- rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
+- rpl->wr.wr_lo = htonl(V_WR_TID(tid));
+- OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
+- rpl->cmd = cmd;
+-}
+-
+-/*
+- * CPL message processing ...
+- * ==========================
+- */
+-
+-/*
+- * Updates connection state from an active establish CPL message. Runs with
+- * the connection lock held.
+- */
+-static void c3cn_act_establish(struct s3_conn *c3cn,
+- struct sk_buff *skb)
+-{
+- struct cpl_act_establish *req = cplhdr(skb);
+- u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
+-
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
+-
+- if (unlikely(c3cn->state != C3CN_STATE_SYN_SENT))
+- printk(KERN_ERR "TID %u expected SYN_SENT, found %d\n",
+- c3cn->tid, c3cn->state);
+-
+- c3cn->copied_seq = c3cn->rcv_wup = c3cn->rcv_nxt = rcv_isn;
+- make_established(c3cn, ntohl(req->snd_isn), ntohs(req->tcp_opt));
+-
+- if (unlikely(c3cn_flag(c3cn, C3CN_CLOSE_NEEDED))) {
+- /* upper layer has requested closing */
+- abort_conn(c3cn, skb);
+- return;
+- }
+-
+- __kfree_skb(skb);
+- if (s3_push_frames(c3cn, 1))
+- cxgb3i_conn_tx_open(c3cn);
+-}
+-
+-/*
+- * Handle active open failures.
+- */
+-static void active_open_failed(struct s3_conn *c3cn,
+- struct sk_buff *skb)
+-{
+- struct cpl_act_open_rpl *rpl = cplhdr(skb);
+-
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
+-
+- if (rpl->status == CPL_ERR_CONN_EXIST &&
+- c3cn->retry_timer.function != act_open_retry_timer) {
+- c3cn->retry_timer.function = act_open_retry_timer;
+- c3cn_reset_timer(c3cn, &c3cn->retry_timer,
+- jiffies + HZ / 2);
+- } else
+- fail_act_open(c3cn, act_open_rpl_status_to_errno(rpl->status));
+- __kfree_skb(skb);
+-}
+-
+-/*
+- * Process received pdu for a connection.
++ * Process RX_ISCSI_HDR CPL message: -> host
++ * Handle received PDUs, the payload could be DDP'ed. If not, the payload
++ * follow after the bhs.
+ */
+-static void process_rx_iscsi_hdr(struct s3_conn *c3cn,
+- struct sk_buff *skb)
++static void process_rx_iscsi_hdr(struct s3_conn *c3cn, struct sk_buff *skb)
+ {
+ struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
+ struct cpl_iscsi_hdr_norss data_cpl;
+@@ -1554,8 +1074,10 @@ static void process_rx_iscsi_hdr(struct
+ unsigned int len;
+ int err;
+
+- if (unlikely(c3cn_no_receive(c3cn))) {
+- handle_excess_rx(c3cn, skb);
++ if (unlikely(c3cn->state >= C3CN_STATE_PASSIVE_CLOSE)) {
++ if (c3cn->state != C3CN_STATE_ABORTING)
++ send_abort_req(c3cn);
++ __kfree_skb(skb);
+ return;
+ }
+
+@@ -1568,8 +1090,9 @@ static void process_rx_iscsi_hdr(struct
+ len = hdr_len = ntohs(hdr_cpl->len);
+ /* msg coalesce is off or not enough data received */
+ if (skb->len <= hdr_len) {
+- printk(KERN_ERR "%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
+- c3cn->cdev->name, c3cn->tid, skb->len, hdr_len);
++ cxgb3i_log_error("%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
++ c3cn->cdev->name, c3cn->tid,
++ skb->len, hdr_len);
+ goto abort_conn;
+ }
+
+@@ -1586,6 +1109,9 @@ static void process_rx_iscsi_hdr(struct
+ c3cn_rx_debug("skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
+ skb, skb->len, skb_ulp_pdulen(skb), status);
+
++ c3cn_rx_debug("rx skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
++ skb, skb->len, skb_ulp_pdulen(skb), status);
++
+ if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
+ skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
+ if (status & (1 << RX_DDP_STATUS_DCRC_SHIFT))
+@@ -1610,15 +1136,24 @@ static void process_rx_iscsi_hdr(struct
+ return;
+
+ abort_conn:
+- s3_send_reset(c3cn, CPL_ABORT_SEND_RST, NULL);
++ send_abort_req(c3cn);
+ __kfree_skb(skb);
+ }
+
++static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
++{
++ struct s3_conn *c3cn = ctx;
++
++ process_cpl_msg(process_rx_iscsi_hdr, c3cn, skb);
++ return 0;
++}
++
+ /*
++ * Process TX_DATA_ACK CPL messages: -> host
+ * Process an acknowledgment of WR completion. Advance snd_una and send the
+ * next batch of work requests from the write queue.
+ */
+-static void wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
++static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
+ {
+ struct cpl_wr_ack *hdr = cplhdr(skb);
+ unsigned int credits = ntohs(hdr->credits);
+@@ -1632,9 +1167,9 @@ static void wr_ack(struct s3_conn *c3cn,
+ struct sk_buff *p = peek_wr(c3cn);
+
+ if (unlikely(!p)) {
+- printk(KERN_ERR "%u WR_ACK credits for TID %u with "
+- "nothing pending, state %u\n",
+- credits, c3cn->tid, c3cn->state);
++ cxgb3i_log_error("%u WR_ACK credits for TID %u with "
++ "nothing pending, state %u\n",
++ credits, c3cn->tid, c3cn->state);
+ break;
+ }
+ if (unlikely(credits < p->csum)) {
+@@ -1653,186 +1188,262 @@ static void wr_ack(struct s3_conn *c3cn,
+ if (c3cn->snd_una != snd_una) {
+ c3cn->snd_una = snd_una;
+ dst_confirm(c3cn->dst_cache);
+- if (c3cn->snd_una == c3cn->snd_nxt)
+- c3cn_reset_flag(c3cn, C3CN_TX_WAIT_IDLE);
+ }
+
+- if (skb_queue_len(&c3cn->write_queue) && s3_push_frames(c3cn, 0))
++ if (skb_queue_len(&c3cn->write_queue) && c3cn_push_tx_frames(c3cn, 0))
+ cxgb3i_conn_tx_open(c3cn);
+ out_free:
+ __kfree_skb(skb);
+ }
+
++static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
++{
++ struct s3_conn *c3cn = ctx;
++
++ process_cpl_msg(process_wr_ack, c3cn, skb);
++ return 0;
++}
++
+ /*
+- * Handle a peer FIN.
++ * for each connection, pre-allocate skbs needed for close/abort requests. So
++ * that we can service the request right away.
+ */
+-static void do_peer_fin(struct s3_conn *c3cn, struct sk_buff *skb)
++static void c3cn_free_cpl_skbs(struct s3_conn *c3cn)
+ {
+- int keep = 0;
+-
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
++ if (c3cn->cpl_close)
++ kfree_skb(c3cn->cpl_close);
++ if (c3cn->cpl_abort_req)
++ kfree_skb(c3cn->cpl_abort_req);
++ if (c3cn->cpl_abort_rpl)
++ kfree_skb(c3cn->cpl_abort_rpl);
++}
+
+- if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
+- goto out;
++static int c3cn_alloc_cpl_skbs(struct s3_conn *c3cn)
++{
++ c3cn->cpl_close = alloc_skb(sizeof(struct cpl_close_con_req),
++ GFP_KERNEL);
++ if (!c3cn->cpl_close)
++ return -ENOMEM;
++ skb_put(c3cn->cpl_close, sizeof(struct cpl_close_con_req));
+
+- c3cn->shutdown |= C3CN_RCV_SHUTDOWN;
+- c3cn_set_flag(c3cn, C3CN_DONE);
++ c3cn->cpl_abort_req = alloc_skb(sizeof(struct cpl_abort_req),
++ GFP_KERNEL);
++ if (!c3cn->cpl_abort_req)
++ goto free_cpl_skbs;
++ skb_put(c3cn->cpl_abort_req, sizeof(struct cpl_abort_req));
++
++ c3cn->cpl_abort_rpl = alloc_skb(sizeof(struct cpl_abort_rpl),
++ GFP_KERNEL);
++ if (!c3cn->cpl_abort_rpl)
++ goto free_cpl_skbs;
++ skb_put(c3cn->cpl_abort_rpl, sizeof(struct cpl_abort_rpl));
+
+- switch (c3cn->state) {
+- case C3CN_STATE_ESTABLISHED:
+- break;
+- case C3CN_STATE_CLOSING:
+- c3cn_done(c3cn);
+- break;
+- default:
+- printk(KERN_ERR
+- "%s: TID %u received PEER_CLOSE in bad state %d\n",
+- c3cn->cdev->name, c3cn->tid, c3cn->state);
+- }
++ return 0;
+
+- cxgb3i_conn_closing(c3cn);
+-out:
+- if (!keep)
+- __kfree_skb(skb);
++free_cpl_skbs:
++ c3cn_free_cpl_skbs(c3cn);
++ return -ENOMEM;
+ }
+
+-/*
+- * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
+- * request except that we need to reply to it.
++/**
++ * c3cn_release_offload_resources - release offload resource
++ * @c3cn: the offloaded iscsi tcp connection.
++ * Release resources held by an offload connection (TID, L2T entry, etc.)
+ */
+-static void process_abort_req(struct s3_conn *c3cn,
+- struct sk_buff *skb)
++static void c3cn_release_offload_resources(struct s3_conn *c3cn)
+ {
+- int rst_status = CPL_ABORT_NO_RST;
+- const struct cpl_abort_req_rss *req = cplhdr(skb);
+-
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
++ struct t3cdev *cdev = c3cn->cdev;
++ unsigned int tid = c3cn->tid;
+
+- if (!c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD)) {
+- c3cn_set_flag(c3cn, C3CN_ABORT_REQ_RCVD);
+- c3cn_set_flag(c3cn, C3CN_ABORT_SHUTDOWN);
+- __kfree_skb(skb);
++ if (!cdev)
+ return;
+- }
+- c3cn_reset_flag(c3cn, C3CN_ABORT_REQ_RCVD);
+
+- /*
+- * Three cases to consider:
+- * a) We haven't sent an abort_req; close the connection.
+- * b) We have sent a post-close abort_req that will get to TP too late
+- * and will generate a CPL_ERR_ABORT_FAILED reply. The reply will
+- * be ignored and the connection should be closed now.
+- * c) We have sent a regular abort_req that will get to TP too late.
+- * That will generate an abort_rpl with status 0, wait for it.
+- */
+- send_abort_rpl(skb, c3cn->cdev, rst_status);
++ c3cn->qset = 0;
+
+- if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
+- c3cn->err =
+- abort_status_to_errno(c3cn, req->status, &rst_status);
++ c3cn_free_cpl_skbs(c3cn);
+
+- c3cn_done(c3cn);
++ if (c3cn->wr_avail != c3cn->wr_max) {
++ purge_wr_queue(c3cn);
++ reset_wr_list(c3cn);
+ }
+-}
+
+-/*
+- * Process abort replies. We only process these messages if we anticipate
+- * them as the coordination between SW and HW in this area is somewhat lacking
+- * and sometimes we get ABORT_RPLs after we are done with the connection that
+- * originated the ABORT_REQ.
+- */
+-static void process_abort_rpl(struct s3_conn *c3cn,
+- struct sk_buff *skb)
+-{
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
++ if (c3cn->l2t) {
++ l2t_release(L2DATA(cdev), c3cn->l2t);
++ c3cn->l2t = NULL;
++ }
+
+- if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
+- if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_RCVD))
+- c3cn_set_flag(c3cn, C3CN_ABORT_RPL_RCVD);
+- else {
+- c3cn_reset_flag(c3cn, C3CN_ABORT_RPL_RCVD);
+- c3cn_reset_flag(c3cn, C3CN_ABORT_RPL_PENDING);
+- BUG_ON(c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD));
+- c3cn_done(c3cn);
+- }
++ if (c3cn->state == C3CN_STATE_CONNECTING) /* we have ATID */
++ s3_free_atid(cdev, tid);
++ else { /* we have TID */
++ cxgb3_remove_tid(cdev, (void *)c3cn, tid);
++ c3cn_put(c3cn);
+ }
+- __kfree_skb(skb);
++
++ c3cn->cdev = NULL;
+ }
+
+-/*
+- * Process a peer ACK to our FIN.
++/**
++ * cxgb3i_c3cn_create - allocate and initialize an s3_conn structure
++ * returns the s3_conn structure allocated.
+ */
+-static void process_close_con_rpl(struct s3_conn *c3cn,
+- struct sk_buff *skb)
++struct s3_conn *cxgb3i_c3cn_create(void)
+ {
+- struct cpl_close_con_rpl *rpl = cplhdr(skb);
++ struct s3_conn *c3cn;
+
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
++ c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL);
++ if (!c3cn)
++ return NULL;
+
+- c3cn->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
++ /* pre-allocate close/abort cpl, so we don't need to wait for memory
++ when close/abort is requested. */
++ if (c3cn_alloc_cpl_skbs(c3cn) < 0)
++ goto free_c3cn;
+
+- if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
+- goto out;
++ c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn);
+
+- if (c3cn->state == C3CN_STATE_CLOSING) {
+- c3cn_done(c3cn);
+- } else
+- printk(KERN_ERR
+- "%s: TID %u received CLOSE_CON_RPL in bad state %d\n",
+- c3cn->cdev->name, c3cn->tid, c3cn->state);
+-out:
+- kfree_skb(skb);
++ c3cn->flags = 0;
++ spin_lock_init(&c3cn->lock);
++ atomic_set(&c3cn->refcnt, 1);
++ skb_queue_head_init(&c3cn->receive_queue);
++ skb_queue_head_init(&c3cn->write_queue);
++ setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn);
++ rwlock_init(&c3cn->callback_lock);
++
++ return c3cn;
++
++free_c3cn:
++ kfree(c3cn);
++ return NULL;
+ }
+
+-/*
+- * Random utility functions for CPL message processing ...
+- * =======================================================
+- */
++static void c3cn_active_close(struct s3_conn *c3cn)
++{
++ int data_lost;
++ int close_req = 0;
++
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ dst_confirm(c3cn->dst_cache);
++
++ c3cn_hold(c3cn);
++ spin_lock_bh(&c3cn->lock);
++
++ data_lost = skb_queue_len(&c3cn->receive_queue);
++ __skb_queue_purge(&c3cn->receive_queue);
++
++ switch (c3cn->state) {
++ case C3CN_STATE_CLOSED:
++ case C3CN_STATE_ACTIVE_CLOSE:
++ case C3CN_STATE_CLOSE_WAIT_1:
++ case C3CN_STATE_CLOSE_WAIT_2:
++ case C3CN_STATE_ABORTING:
++ /* nothing need to be done */
++ break;
++ case C3CN_STATE_CONNECTING:
++ /* defer until cpl_act_open_rpl or cpl_act_establish */
++ c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
++ break;
++ case C3CN_STATE_ESTABLISHED:
++ close_req = 1;
++ c3cn_set_state(c3cn, C3CN_STATE_ACTIVE_CLOSE);
++ break;
++ case C3CN_STATE_PASSIVE_CLOSE:
++ close_req = 1;
++ c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
++ break;
++ }
++
++ if (close_req) {
++ if (data_lost)
++ /* Unread data was tossed, zap the connection. */
++ send_abort_req(c3cn);
++ else
++ send_close_req(c3cn);
++ }
++
++ spin_unlock_bh(&c3cn->lock);
++ c3cn_put(c3cn);
++}
+
+ /**
+- * find_best_mtu - find the entry in the MTU table closest to an MTU
+- * @d: TOM state
+- * @mtu: the target MTU
+- *
+- * Returns the index of the value in the MTU table that is closest to but
+- * does not exceed the target MTU.
++ * cxgb3i_c3cn_release - close and release an iscsi tcp connection and any
++ * resource held
++ * @c3cn: the iscsi tcp connection
+ */
+-static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
++void cxgb3i_c3cn_release(struct s3_conn *c3cn)
+ {
+- int i = 0;
+-
+- while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
+- ++i;
+- return i;
++ c3cn_conn_debug("c3cn 0x%p, s %u, f 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++ if (likely(c3cn->state != C3CN_STATE_CONNECTING))
++ c3cn_active_close(c3cn);
++ else
++ c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
++ c3cn_put(c3cn);
+ }
+
+-static unsigned int select_mss(struct s3_conn *c3cn, unsigned int pmtu)
++static int is_cxgb3_dev(struct net_device *dev)
+ {
+- unsigned int idx;
+- struct dst_entry *dst = c3cn->dst_cache;
+- struct t3cdev *cdev = c3cn->cdev;
+- const struct t3c_data *td = T3C_DATA(cdev);
+- u16 advmss = dst_metric(dst, RTAX_ADVMSS);
++ struct cxgb3i_sdev_data *cdata;
+
+- if (advmss > pmtu - 40)
+- advmss = pmtu - 40;
+- if (advmss < td->mtus[0] - 40)
+- advmss = td->mtus[0] - 40;
+- idx = find_best_mtu(td, advmss + 40);
+- return idx;
++ write_lock(&cdata_rwlock);
++ list_for_each_entry(cdata, &cdata_list, list) {
++ struct adap_ports *ports = &cdata->ports;
++ int i;
++
++ for (i = 0; i < ports->nports; i++)
++ if (dev == ports->lldevs[i]) {
++ write_unlock(&cdata_rwlock);
++ return 1;
++ }
++ }
++ write_unlock(&cdata_rwlock);
++ return 0;
+ }
+
+-static void fail_act_open(struct s3_conn *c3cn, int errno)
++/**
++ * cxgb3_egress_dev - return the cxgb3 egress device
++ * @root_dev: the root device anchoring the search
++ * @c3cn: the connection used to determine egress port in bonding mode
++ * @context: in bonding mode, indicates a connection set up or failover
++ *
++ * Return egress device or NULL if the egress device isn't one of our ports.
++ */
++static struct net_device *cxgb3_egress_dev(struct net_device *root_dev,
++ struct s3_conn *c3cn,
++ int context)
+ {
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
+- c3cn, c3cn->state, c3cn->flags);
++ while (root_dev) {
++ if (root_dev->priv_flags & IFF_802_1Q_VLAN)
++ root_dev = vlan_dev_real_dev(root_dev);
++ else if (is_cxgb3_dev(root_dev))
++ return root_dev;
++ else
++ return NULL;
++ }
++ return NULL;
++}
+
+- c3cn->err = errno;
+- c3cn_done(c3cn);
++static struct rtable *find_route(__be32 saddr, __be32 daddr,
++ __be16 sport, __be16 dport)
++{
++ struct rtable *rt;
++ struct flowi fl = {
++ .oif = 0,
++ .nl_u = {
++ .ip4_u = {
++ .daddr = daddr,
++ .saddr = saddr,
++ .tos = 0 } },
++ .proto = IPPROTO_TCP,
++ .uli_u = {
++ .ports = {
++ .sport = sport,
++ .dport = dport } } };
++
++ if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
++ return NULL;
++ return rt;
+ }
+
+ /*
+@@ -1847,195 +1458,355 @@ static void init_offload_conn(struct s3_
+ c3cn->wr_unacked = 0;
+ c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
+
+- c3cn->ctrl_skb_cache = alloc_skb(CTRL_SKB_LEN, gfp_any());
+ reset_wr_list(c3cn);
+ }
+
+-static void act_open_retry_timer(unsigned long data)
++static int initiate_act_open(struct s3_conn *c3cn, struct net_device *dev)
+ {
++ struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev);
++ struct t3cdev *cdev = cdata->cdev;
++ struct dst_entry *dst = c3cn->dst_cache;
+ struct sk_buff *skb;
+- struct s3_conn *c3cn = (struct s3_conn *)data;
+
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
++ c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++ /*
++ * Initialize connection data. Note that the flags and ULP mode are
++ * initialized higher up ...
++ */
++ c3cn->dev = dev;
++ c3cn->cdev = cdev;
++ c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn);
++ if (c3cn->tid < 0)
++ goto out_err;
+
+- spin_lock(&c3cn->lock);
+- skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_ATOMIC);
++ c3cn->qset = 0;
++ c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev);
++ if (!c3cn->l2t)
++ goto free_tid;
++
++ skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL);
+ if (!skb)
+- fail_act_open(c3cn, ENOMEM);
+- else {
+- skb->sk = (struct sock *)c3cn;
+- set_arp_failure_handler(skb, act_open_req_arp_failure);
+- mk_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
+- l2t_send(c3cn->cdev, skb, c3cn->l2t);
+- }
+- spin_unlock(&c3cn->lock);
+- c3cn_put(c3cn);
+-}
++ goto free_l2t;
+
+-/*
+- * Convert an ACT_OPEN_RPL status to a Linux errno.
+- */
+-static int act_open_rpl_status_to_errno(int status)
+-{
+- switch (status) {
+- case CPL_ERR_CONN_RESET:
+- return ECONNREFUSED;
+- case CPL_ERR_ARP_MISS:
+- return EHOSTUNREACH;
+- case CPL_ERR_CONN_TIMEDOUT:
+- return ETIMEDOUT;
+- case CPL_ERR_TCAM_FULL:
+- return ENOMEM;
+- case CPL_ERR_CONN_EXIST:
+- printk(KERN_ERR "ACTIVE_OPEN_RPL: 4-tuple in use\n");
+- return EADDRINUSE;
+- default:
+- return EIO;
+- }
++ skb->sk = (struct sock *)c3cn;
++ set_arp_failure_handler(skb, act_open_req_arp_failure);
++
++ c3cn_hold(c3cn);
++
++ init_offload_conn(c3cn, cdev, dst);
++ c3cn->err = 0;
++
++ make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
++ l2t_send(cdev, skb, c3cn->l2t);
++ return 0;
++
++free_l2t:
++ l2t_release(L2DATA(cdev), c3cn->l2t);
++free_tid:
++ s3_free_atid(cdev, c3cn->tid);
++ c3cn->tid = 0;
++out_err:
++ return -1;
+ }
+
+-/*
+- * Convert the status code of an ABORT_REQ into a Linux error code. Also
+- * indicate whether RST should be sent in response.
++
++/**
++ * cxgb3i_c3cn_connect - initiates an iscsi tcp connection to a given address
++ * @c3cn: the iscsi tcp connection
++ * @usin: destination address
++ *
++ * return 0 if active open request is sent, < 0 otherwise.
+ */
+-static int abort_status_to_errno(struct s3_conn *c3cn,
+- int abort_reason, int *need_rst)
++int cxgb3i_c3cn_connect(struct s3_conn *c3cn, struct sockaddr_in *usin)
+ {
+- switch (abort_reason) {
+- case CPL_ERR_BAD_SYN: /* fall through */
+- case CPL_ERR_CONN_RESET:
+- return c3cn->state == C3CN_STATE_CLOSING ? EPIPE : ECONNRESET;
+- case CPL_ERR_XMIT_TIMEDOUT:
+- case CPL_ERR_PERSIST_TIMEDOUT:
+- case CPL_ERR_FINWAIT2_TIMEDOUT:
+- case CPL_ERR_KEEPALIVE_TIMEDOUT:
+- return ETIMEDOUT;
+- default:
+- return EIO;
++ struct rtable *rt;
++ struct net_device *dev;
++ struct cxgb3i_sdev_data *cdata;
++ struct t3cdev *cdev;
++ __be32 sipv4;
++ int err;
++
++ if (usin->sin_family != AF_INET)
++ return -EAFNOSUPPORT;
++
++ c3cn->daddr.sin_port = usin->sin_port;
++ c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr;
++
++ rt = find_route(c3cn->saddr.sin_addr.s_addr,
++ c3cn->daddr.sin_addr.s_addr,
++ c3cn->saddr.sin_port,
++ c3cn->daddr.sin_port);
++ if (rt == NULL) {
++ c3cn_conn_debug("NO route to 0x%x, port %u.\n",
++ c3cn->daddr.sin_addr.s_addr,
++ ntohs(c3cn->daddr.sin_port));
++ return -ENETUNREACH;
+ }
+-}
+
+-static void send_abort_rpl(struct sk_buff *skb, struct t3cdev *cdev,
+- int rst_status)
+-{
+- struct sk_buff *reply_skb;
+- struct cpl_abort_req_rss *req = cplhdr(skb);
++ if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
++ c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n",
++ c3cn->daddr.sin_addr.s_addr,
++ ntohs(c3cn->daddr.sin_port));
++ ip_rt_put(rt);
++ return -ENETUNREACH;
++ }
+
+- reply_skb = get_cpl_reply_skb(skb, sizeof(struct cpl_abort_rpl),
+- gfp_any());
++ if (!c3cn->saddr.sin_addr.s_addr)
++ c3cn->saddr.sin_addr.s_addr = rt->rt_src;
+
+- reply_skb->priority = CPL_PRIORITY_DATA;
+- set_abort_rpl_wr(reply_skb, GET_TID(req), rst_status);
+- kfree_skb(skb);
+- cxgb3_ofld_send(cdev, reply_skb);
+-}
++ /* now commit destination to connection */
++ c3cn->dst_cache = &rt->u.dst;
+
+-/*
+- * Returns an sk_buff for a reply CPL message of size len. If the input
+- * sk_buff has no other users it is trimmed and reused, otherwise a new buffer
+- * is allocated. The input skb must be of size at least len. Note that this
+- * operation does not destroy the original skb data even if it decides to reuse
+- * the buffer.
+- */
+-static struct sk_buff *get_cpl_reply_skb(struct sk_buff *skb, size_t len,
+- gfp_t gfp)
+-{
+- if (likely(!skb_cloned(skb))) {
+- BUG_ON(skb->len < len);
+- __skb_trim(skb, len);
+- skb_get(skb);
+- } else {
+- skb = alloc_skb(len, gfp);
+- if (skb)
+- __skb_put(skb, len);
++ /* try to establish an offloaded connection */
++ dev = cxgb3_egress_dev(c3cn->dst_cache->dev, c3cn, 0);
++ if (dev == NULL) {
++ c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn);
++ return -ENETUNREACH;
+ }
+- return skb;
++ cdata = NDEV2CDATA(dev);
++ cdev = cdata->cdev;
++
++ /* get a source port if one hasn't been provided */
++ err = c3cn_get_port(c3cn, cdata);
++ if (err)
++ return err;
++
++ c3cn_conn_debug("c3cn 0x%p get port %u.\n",
++ c3cn, ntohs(c3cn->saddr.sin_port));
++
++ sipv4 = cxgb3i_get_private_ipv4addr(dev);
++ if (!sipv4) {
++ c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn);
++ sipv4 = c3cn->saddr.sin_addr.s_addr;
++ cxgb3i_set_private_ipv4addr(dev, sipv4);
++ } else
++ c3cn->saddr.sin_addr.s_addr = sipv4;
++
++ c3cn_conn_debug("c3cn 0x%p, %u.%u.%u.%u,%u-%u.%u.%u.%u,%u SYN_SENT.\n",
++ c3cn, NIPQUAD(c3cn->saddr.sin_addr.s_addr),
++ ntohs(c3cn->saddr.sin_port),
++ NIPQUAD(c3cn->daddr.sin_addr.s_addr),
++ ntohs(c3cn->daddr.sin_port));
++
++ c3cn_set_state(c3cn, C3CN_STATE_CONNECTING);
++ if (!initiate_act_open(c3cn, dev))
++ return 0;
++
++ /*
++ * If we get here, we don't have an offload connection so simply
++ * return a failure.
++ */
++ err = -ENOTSUPP;
++
++ /*
++ * This trashes the connection and releases the local port,
++ * if necessary.
++ */
++ c3cn_conn_debug("c3cn 0x%p -> CLOSED.\n", c3cn);
++ c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
++ ip_rt_put(rt);
++ c3cn_put_port(c3cn);
++ c3cn->daddr.sin_port = 0;
++ return err;
+ }
+
+-/*
+- * Release resources held by an offload connection (TID, L2T entry, etc.)
++/**
++ * cxgb3i_c3cn_rx_credits - ack received tcp data.
++ * @c3cn: iscsi tcp connection
++ * @copied: # of bytes processed
++ *
++ * Called after some received data has been read. It returns RX credits
++ * to the HW for the amount of data processed.
+ */
+-static void t3_release_offload_resources(struct s3_conn *c3cn)
++void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied)
+ {
+- struct t3cdev *cdev = c3cn->cdev;
+- unsigned int tid = c3cn->tid;
++ struct t3cdev *cdev;
++ int must_send;
++ u32 credits, dack = 0;
+
+- if (!cdev)
++ if (c3cn->state != C3CN_STATE_ESTABLISHED)
+ return;
+
+- c3cn->qset = 0;
++ credits = c3cn->copied_seq - c3cn->rcv_wup;
++ if (unlikely(!credits))
++ return;
+
+- kfree_skb(c3cn->ctrl_skb_cache);
+- c3cn->ctrl_skb_cache = NULL;
++ cdev = c3cn->cdev;
+
+- if (c3cn->wr_avail != c3cn->wr_max) {
+- purge_wr_queue(c3cn);
+- reset_wr_list(c3cn);
++ if (unlikely(cxgb3_rx_credit_thres == 0))
++ return;
++
++ dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
++
++ /*
++ * For coalescing to work effectively ensure the receive window has
++ * at least 16KB left.
++ */
++ must_send = credits + 16384 >= cxgb3_rcv_win;
++
++ if (must_send || credits >= cxgb3_rx_credit_thres)
++ c3cn->rcv_wup += send_rx_credits(c3cn, credits, dack);
++}
++
++/**
++ * cxgb3i_c3cn_send_pdus - send the skbs containing iscsi pdus
++ * @c3cn: iscsi tcp connection
++ * @skb: skb contains the iscsi pdu
++ *
++ * Add a list of skbs to a connection send queue. The skbs must comply with
++ * the max size limit of the device and have a headroom of at least
++ * TX_HEADER_LEN bytes.
++ * Return # of bytes queued.
++ */
++int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb)
++{
++ struct sk_buff *next;
++ int err, copied = 0;
++
++ spin_lock_bh(&c3cn->lock);
++
++ if (c3cn->state != C3CN_STATE_ESTABLISHED) {
++ c3cn_tx_debug("c3cn 0x%p, not in est. state %u.\n",
++ c3cn, c3cn->state);
++ err = -EAGAIN;
++ goto out_err;
+ }
+
+- if (c3cn->l2t) {
+- l2t_release(L2DATA(cdev), c3cn->l2t);
+- c3cn->l2t = NULL;
++ err = -EPIPE;
++ if (c3cn->err) {
++ c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn, c3cn->err);
++ goto out_err;
+ }
+
+- if (c3cn->state == C3CN_STATE_SYN_SENT) /* we have ATID */
+- free_atid(cdev, tid);
+- else { /* we have TID */
+- cxgb3_remove_tid(cdev, (void *)c3cn, tid);
+- c3cn_put(c3cn);
++ while (skb) {
++ int frags = skb_shinfo(skb)->nr_frags +
++ (skb->len != skb->data_len);
++
++ if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
++ c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
++ err = -EINVAL;
++ goto out_err;
++ }
++
++ if (frags >= SKB_WR_LIST_SIZE) {
++ cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
++ c3cn, skb_shinfo(skb)->nr_frags,
++ skb->len, skb->data_len);
++ err = -EINVAL;
++ goto out_err;
++ }
++
++ next = skb->next;
++ skb->next = NULL;
++ skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
++ copied += skb->len;
++ c3cn->write_seq += skb->len + ulp_extra_len(skb);
++ skb = next;
+ }
++done:
++ if (likely(skb_queue_len(&c3cn->write_queue)))
++ c3cn_push_tx_frames(c3cn, 1);
++ spin_unlock_bh(&c3cn->lock);
++ return copied;
+
+- c3cn->cdev = NULL;
++out_err:
++ if (copied == 0 && err == -EPIPE)
++ copied = c3cn->err ? c3cn->err : -EPIPE;
++ goto done;
+ }
+
+-/*
+- * Handles Rx data that arrives in a state where the connection isn't
+- * accepting new data.
+- */
+-static void handle_excess_rx(struct s3_conn *c3cn, struct sk_buff *skb)
++static void sdev_data_cleanup(struct cxgb3i_sdev_data *cdata)
+ {
+- if (!c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN))
+- abort_conn(c3cn, skb);
++ struct adap_ports *ports = &cdata->ports;
++ int i;
+
+- kfree_skb(skb);
++ for (i = 0; i < ports->nports; i++)
++ NDEV2CDATA(ports->lldevs[i]) = NULL;
++ cxgb3i_free_big_mem(cdata);
+ }
+
+-/*
+- * Like get_cpl_reply_skb() but the returned buffer starts out empty.
+- */
+-static struct sk_buff *__get_cpl_reply_skb(struct sk_buff *skb, size_t len,
+- gfp_t gfp)
++void cxgb3i_sdev_cleanup(void)
+ {
+- if (likely(!skb_cloned(skb) && !skb->data_len)) {
+- __skb_trim(skb, 0);
+- skb_get(skb);
+- } else
+- skb = alloc_skb(len, gfp);
+- return skb;
++ struct cxgb3i_sdev_data *cdata;
++
++ write_lock(&cdata_rwlock);
++ list_for_each_entry(cdata, &cdata_list, list) {
++ list_del(&cdata->list);
++ sdev_data_cleanup(cdata);
++ }
++ write_unlock(&cdata_rwlock);
+ }
+
+-/*
+- * Completes some final bits of initialization for just established connections
+- * and changes their state to C3CN_STATE_ESTABLISHED.
+- *
+- * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
++int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers)
++{
++ cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish;
++ cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl;
++ cpl_handlers[CPL_PEER_CLOSE] = do_peer_close;
++ cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req;
++ cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl;
++ cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl;
++ cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack;
++ cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr;
++
++ if (cxgb3_max_connect > CXGB3I_MAX_CONN)
++ cxgb3_max_connect = CXGB3I_MAX_CONN;
++ return 0;
++}
++
++/**
++ * cxgb3i_sdev_add - allocate and initialize resources for each adapter found
++ * @cdev: t3cdev adapter
++ * @client: cxgb3 driver client
+ */
+-static void make_established(struct s3_conn *c3cn, u32 snd_isn,
+- unsigned int opt)
++void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client)
+ {
+- c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
++ struct cxgb3i_sdev_data *cdata;
++ struct ofld_page_info rx_page_info;
++ unsigned int wr_len;
++ int mapsize = DIV_ROUND_UP(cxgb3_max_connect,
++ 8 * sizeof(unsigned long));
++ int i;
+
+- c3cn->write_seq = c3cn->snd_nxt = c3cn->snd_una = snd_isn;
++ cdata = cxgb3i_alloc_big_mem(sizeof(*cdata) + mapsize, GFP_KERNEL);
++ if (!cdata)
++ return;
+
+- /*
+- * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
+- * pass through opt0.
+- */
+- if (cxgb3_rcv_win > (M_RCV_BUFSIZ << 10))
+- c3cn->rcv_wup -= cxgb3_rcv_win - (M_RCV_BUFSIZ << 10);
++ if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 ||
++ cdev->ctl(cdev, GET_PORTS, &cdata->ports) < 0 ||
++ cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0)
++ goto free_cdata;
+
+- dst_confirm(c3cn->dst_cache);
++ s3_init_wr_tab(wr_len);
+
+- smp_mb();
+- c3cn_set_state(c3cn, C3CN_STATE_ESTABLISHED);
++ INIT_LIST_HEAD(&cdata->list);
++ cdata->cdev = cdev;
++ cdata->client = client;
++
++ for (i = 0; i < cdata->ports.nports; i++)
++ NDEV2CDATA(cdata->ports.lldevs[i]) = cdata;
++
++ write_lock(&cdata_rwlock);
++ list_add_tail(&cdata->list, &cdata_list);
++ write_unlock(&cdata_rwlock);
++
++ return;
++
++free_cdata:
++ cxgb3i_free_big_mem(cdata);
++}
++
++/**
++ * cxgb3i_sdev_remove - free the allocated resources for the adapter
++ * @cdev: t3cdev adapter
++ */
++void cxgb3i_sdev_remove(struct t3cdev *cdev)
++{
++ struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
++
++ write_lock(&cdata_rwlock);
++ list_del(&cdata->list);
++ write_unlock(&cdata_rwlock);
++
++ sdev_data_cleanup(cdata);
+ }
+--- a/drivers/scsi/cxgb3i/cxgb3i_offload.h
++++ b/drivers/scsi/cxgb3i/cxgb3i_offload.h
+@@ -1,12 +1,15 @@
+ /*
+- * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
++ * cxgb3i_offload.h: Chelsio S3xx iscsi offloaded tcp connection management
+ *
+- * Written by Dimitris Michailidis (dm@chelsio.com)
++ * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
+ * release for licensing terms and conditions.
++ *
++ * Written by: Dimitris Michailidis (dm@chelsio.com)
++ * Karen Xie (kxie@chelsio.com)
+ */
+
+ #ifndef _CXGB3I_OFFLOAD_H
+@@ -23,83 +26,104 @@
+ #define cxgb3i_log_error(fmt...) printk(KERN_ERR "cxgb3i: ERR! " fmt)
+ #define cxgb3i_log_warn(fmt...) printk(KERN_WARNING "cxgb3i: WARN! " fmt)
+ #define cxgb3i_log_info(fmt...) printk(KERN_INFO "cxgb3i: " fmt)
+-
+-#ifdef __DEBUG_CXGB3I__
+ #define cxgb3i_log_debug(fmt, args...) \
+ printk(KERN_INFO "cxgb3i: %s - " fmt, __func__ , ## args)
+-#else
+-#define cxgb3i_log_debug(fmt...)
+-#endif
+-
+-#ifdef __DEBUG_C3CN_CONN__
+-#define c3cn_conn_debug cxgb3i_log_debug
+-#else
+-#define c3cn_conn_debug(fmt...)
+-#endif
+
+-/*
+- * Data structure to keep track of cxgb3 connection.
++/**
++ * struct s3_conn - an iscsi tcp connection structure
++ *
++ * @dev: net device of with connection
++ * @cdev: adapter t3cdev for net device
++ * @flags: see c3cn_flags below
++ * @tid: connection id assigned by the h/w
++ * @qset: queue set used by connection
++ * @mss_idx: Maximum Segment Size table index
++ * @l2t: ARP resolution entry for offload packets
++ * @wr_max: maximum in-flight writes
++ * @wr_avail: number of writes available
++ * @wr_unacked: writes since last request for completion notification
++ * @wr_pending_head: head of pending write queue
++ * @wr_pending_tail: tail of pending write queue
++ * @cpl_close: skb for cpl_close_req
++ * @cpl_abort_req: skb for cpl_abort_req
++ * @cpl_abort_rpl: skb for cpl_abort_rpl
++ * @lock: connection status lock
++ * @refcnt: reference count on connection
++ * @state: connection state
++ * @saddr: source ip/port address
++ * @daddr: destination ip/port address
++ * @dst_cache: reference to destination route
++ * @receive_queue: received PDUs
++ * @write_queue: un-pushed pending writes
++ * @retry_timer: retry timer for various operations
++ * @err: connection error status
++ * @callback_lock: lock for opaque user context
++ * @user_data: opaque user context
++ * @rcv_nxt: next receive seq. #
++ * @copied_seq: head of yet unread data
++ * @rcv_wup: rcv_nxt on last window update sent
++ * @snd_nxt: next sequence we send
++ * @snd_una: first byte we want an ack for
++ * @write_seq: tail+1 of data held in send buffer
+ */
+ struct s3_conn {
+- struct net_device *dev; /* net device of with connection */
+- struct t3cdev *cdev; /* adapter t3cdev for net device */
+- unsigned long flags; /* see c3cn_flags below */
+- int tid; /* ID of TCP Control Block */
+- int qset; /* queue Set used by connection */
+- int mss_idx; /* Maximum Segment Size table index */
+- struct l2t_entry *l2t; /* ARP resolution for offload packets */
+- int wr_max; /* maximum in-flight writes */
+- int wr_avail; /* number of writes available */
+- int wr_unacked; /* writes since last request for */
+- /* completion notification */
+- struct sk_buff *wr_pending_head;/* head of pending write queue */
+- struct sk_buff *wr_pending_tail;/* tail of pending write queue */
+- struct sk_buff *ctrl_skb_cache; /* single entry cached skb for */
+- /* short-term control operations */
+- spinlock_t lock; /* connection status lock */
+- atomic_t refcnt; /* reference count on connection */
+- volatile unsigned int state; /* connection state */
+- struct sockaddr_in saddr; /* source IP/port address */
+- struct sockaddr_in daddr; /* destination IP/port address */
+- struct dst_entry *dst_cache; /* reference to destination route */
+- unsigned char shutdown; /* shutdown status */
+- struct sk_buff_head receive_queue;/* received PDUs */
+- struct sk_buff_head write_queue;/* un-pushed pending writes */
+-
+- struct timer_list retry_timer; /* retry timer for various operations */
+- int err; /* connection error status */
+- rwlock_t callback_lock; /* lock for opaque user context */
+- void *user_data; /* opaque user context */
+-
+- u32 rcv_nxt; /* what we want to receive next */
+- u32 copied_seq; /* head of yet unread data */
+- u32 rcv_wup; /* rcv_nxt on last window update sent */
+- u32 snd_nxt; /* next sequence we send */
+- u32 snd_una; /* first byte we want an ack for */
+-
+- u32 write_seq; /* tail+1 of data held in send buffer */
+-};
+-
+-/* Flags in c3cn->shutdown */
+-#define C3CN_RCV_SHUTDOWN 0x1
+-#define C3CN_SEND_SHUTDOWN 0x2
+-#define C3CN_SHUTDOWN_MASK (C3CN_RCV_SHUTDOWN | C3CN_SEND_SHUTDOWN)
++ struct net_device *dev;
++ struct t3cdev *cdev;
++ unsigned long flags;
++ int tid;
++ int qset;
++ int mss_idx;
++ struct l2t_entry *l2t;
++ int wr_max;
++ int wr_avail;
++ int wr_unacked;
++ struct sk_buff *wr_pending_head;
++ struct sk_buff *wr_pending_tail;
++ struct sk_buff *cpl_close;
++ struct sk_buff *cpl_abort_req;
++ struct sk_buff *cpl_abort_rpl;
++ spinlock_t lock;
++ atomic_t refcnt;
++ volatile unsigned int state;
++ struct sockaddr_in saddr;
++ struct sockaddr_in daddr;
++ struct dst_entry *dst_cache;
++ struct sk_buff_head receive_queue;
++ struct sk_buff_head write_queue;
++ struct timer_list retry_timer;
++ int err;
++ rwlock_t callback_lock;
++ void *user_data;
++
++ u32 rcv_nxt;
++ u32 copied_seq;
++ u32 rcv_wup;
++ u32 snd_nxt;
++ u32 snd_una;
++ u32 write_seq;
++};
+
+ /*
+- * connection state bitmap
+- */
+-#define C3CN_STATE_CLOSE 0x1
+-#define C3CN_STATE_SYN_SENT 0x2
+-#define C3CN_STATE_ESTABLISHED 0x4
+-#define C3CN_STATE_CLOSING 0x8
+-#define C3CN_STATE_ABORING 0x10
+-
+-#define C3CN_STATE_MASK 0xFF
++ * connection state
++ */
++enum conn_states {
++ C3CN_STATE_CONNECTING = 1,
++ C3CN_STATE_ESTABLISHED,
++ C3CN_STATE_ACTIVE_CLOSE,
++ C3CN_STATE_PASSIVE_CLOSE,
++ C3CN_STATE_CLOSE_WAIT_1,
++ C3CN_STATE_CLOSE_WAIT_2,
++ C3CN_STATE_ABORTING,
++ C3CN_STATE_CLOSED,
++};
+
+-static inline unsigned int c3cn_in_state(const struct s3_conn *c3cn,
+- unsigned int states)
++static inline unsigned int c3cn_is_closing(const struct s3_conn *c3cn)
+ {
+- return states & c3cn->state;
++ return c3cn->state >= C3CN_STATE_ACTIVE_CLOSE;
++}
++static inline unsigned int c3cn_is_established(const struct s3_conn *c3cn)
++{
++ return c3cn->state == C3CN_STATE_ESTABLISHED;
+ }
+
+ /*
+@@ -108,37 +132,35 @@ static inline unsigned int c3cn_in_state
+ enum c3cn_flags {
+ C3CN_ABORT_RPL_RCVD, /* received one ABORT_RPL_RSS message */
+ C3CN_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
+- C3CN_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
+- C3CN_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
+-
+ C3CN_ABORT_RPL_PENDING, /* expecting an abort reply */
+- C3CN_CLOSE_CON_REQUESTED, /* we've sent a close_conn_req */
+ C3CN_TX_DATA_SENT, /* already sent a TX_DATA WR */
+- C3CN_CLOSE_NEEDED, /* need to be closed */
+- C3CN_DONE,
++ C3CN_ACTIVE_CLOSE_NEEDED, /* need to be closed */
+ };
+
+-/*
+- * Per adapter data. Linked off of each Ethernet device port on the adapter.
++/**
++ * cxgb3i_sdev_data - Per adapter data.
++ * Linked off of each Ethernet device port on the adapter.
+ * Also available via the t3cdev structure since we have pointers to our port
+ * net_device's there ...
++ *
++ * @list: list head to link elements
++ * @cdev: t3cdev adapter
++ * @client: CPL client pointer
++ * @ports: array of adapter ports
++ * @sport_map_next: next index into the port map
++ * @sport_map: source port map
+ */
+ struct cxgb3i_sdev_data {
+- struct list_head list; /* links for list of all adapters */
+- struct t3cdev *cdev; /* adapter t3cdev */
+- struct cxgb3_client *client; /* CPL client pointer */
+- struct adap_ports *ports; /* array of adapter ports */
+- unsigned int rx_page_size; /* RX page size */
+- struct sk_buff_head deferq; /* queue for processing replies from */
+- /* worker thread context */
+- struct work_struct deferq_task; /* worker thread */
++ struct list_head list;
++ struct t3cdev *cdev;
++ struct cxgb3_client *client;
++ struct adap_ports ports;
++ unsigned int sport_map_next;
++ unsigned long sport_map[0];
+ };
+ #define NDEV2CDATA(ndev) (*(struct cxgb3i_sdev_data **)&(ndev)->ec_ptr)
+ #define CXGB3_SDEV_DATA(cdev) NDEV2CDATA((cdev)->lldev)
+
+-/*
+- * Primary API routines.
+- */
+ void cxgb3i_sdev_cleanup(void);
+ int cxgb3i_sdev_init(cxgb3_cpl_handler_func *);
+ void cxgb3i_sdev_add(struct t3cdev *, struct cxgb3_client *);
+@@ -147,20 +169,26 @@ void cxgb3i_sdev_remove(struct t3cdev *)
+ struct s3_conn *cxgb3i_c3cn_create(void);
+ int cxgb3i_c3cn_connect(struct s3_conn *, struct sockaddr_in *);
+ void cxgb3i_c3cn_rx_credits(struct s3_conn *, int);
+-int cxgb3i_c3cn_send_pdus(struct s3_conn *, struct sk_buff *, int);
++int cxgb3i_c3cn_send_pdus(struct s3_conn *, struct sk_buff *);
+ void cxgb3i_c3cn_release(struct s3_conn *);
+
+-/*
+- * Definitions for sk_buff state and ULP mode management.
++/**
++ * cxgb3_skb_cb - control block for received pdu state and ULP mode management.
++ *
++ * @flag: see C3CB_FLAG_* below
++ * @ulp_mode: ULP mode/submode of sk_buff
++ * @seq: tcp sequence number
++ * @ddigest: pdu data digest
++ * @pdulen: recovered pdu length
++ * @ulp_data: scratch area for ULP
+ */
+-
+ struct cxgb3_skb_cb {
+- __u8 flags; /* see C3CB_FLAG_* below */
+- __u8 ulp_mode; /* ULP mode/submode of sk_buff */
+- __u32 seq; /* sequence number */
+- __u32 ddigest; /* ULP rx_data_ddp selected field */
+- __u32 pdulen; /* ULP rx_data_ddp selected field */
+- __u8 ulp_data[16]; /* scratch area for ULP */
++ __u8 flags;
++ __u8 ulp_mode;
++ __u32 seq;
++ __u32 ddigest;
++ __u32 pdulen;
++ __u8 ulp_data[16];
+ };
+
+ #define CXGB3_SKB_CB(skb) ((struct cxgb3_skb_cb *)&((skb)->cb[0]))
+@@ -170,28 +198,14 @@ struct cxgb3_skb_cb {
+ #define skb_ulp_pdulen(skb) (CXGB3_SKB_CB(skb)->pdulen)
+ #define skb_ulp_data(skb) (CXGB3_SKB_CB(skb)->ulp_data)
+
+-enum {
++enum c3cb_flags {
+ C3CB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
+ C3CB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
+- C3CB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */
+- C3CB_FLAG_COMPL = 1 << 4, /* request WR completion */
++ C3CB_FLAG_COMPL = 1 << 2, /* request WR completion */
+ };
+
+-/*
+- * Top-level CPL message processing used by most CPL messages that
+- * pertain to connections.
+- */
+-static inline void process_cpl_msg(void (*fn)(struct s3_conn *,
+- struct sk_buff *),
+- struct s3_conn *c3cn,
+- struct sk_buff *skb)
+-{
+- spin_lock(&c3cn->lock);
+- fn(c3cn, skb);
+- spin_unlock(&c3cn->lock);
+-}
+-
+-/*
++/**
++ * sge_opaque_hdr -
+ * Opaque version of structure the SGE stores at skb->head of TX_DATA packets
+ * and for which we must reserve space.
+ */
+@@ -204,9 +218,6 @@ struct sge_opaque_hdr {
+ #define TX_HEADER_LEN \
+ (sizeof(struct tx_data_wr) + sizeof(struct sge_opaque_hdr))
+
+-void *cxgb3i_alloc_big_mem(unsigned int);
+-void cxgb3i_free_big_mem(void *);
+-
+ /*
+ * get and set private ip for iscsi traffic
+ */
+--- a/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
++++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
+@@ -312,6 +312,7 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i
+ page_idx, sgcnt, xferlen, ULP2_DDP_THRESHOLD);
+ return RESERVED_ITT;
+ }
++ return RESERVED_ITT;
+
+ gl = ddp_make_gl(xferlen, sgl, sgcnt, gfp);
+ if (!gl) {
+@@ -380,8 +381,14 @@ void cxgb3i_ddp_tag_release(struct cxgb3
+ if (idx < snic->tag_format.rsvd_mask) {
+ struct cxgb3i_ddp_info *ddp = snic->ddp;
+ struct cxgb3i_gather_list *gl = ddp->gl_map[idx];
+- unsigned int npods = (gl->nelem + PPOD_PAGES_MAX - 1) >>
+- PPOD_PAGES_SHIFT;
++ unsigned int npods;
++
++ if (!gl || !gl->nelem) {
++ cxgb3i_log_warn("release tag 0x%x, idx 0x%x, no gl.\n",
++ tag, idx);
++ return;
++ }
++ npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
+
+ cxgb3i_tag_debug("ddp tag 0x%x, release idx 0x%x, npods %u.\n",
+ tag, idx, npods);
+@@ -469,14 +476,14 @@ static int cxgb3i_conn_read_pdu_skb(stru
+ (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) ?
+ ISCSI_SEGMENT_DGST_ERR : 0;
+ if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
+- cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, ddp'ed, "
+- "itt 0x%x.\n",
++ cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, "
++ "ddp'ed, itt 0x%x.\n",
+ skb, hdr->opcode & ISCSI_OPCODE_MASK,
+ tcp_conn->in.datalen, hdr->itt);
+ segment->total_copied = segment->total_size;
+ } else {
+- cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, not ddp'ed, "
+- "itt 0x%x.\n",
++ cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, "
++ "not ddp'ed, itt 0x%x.\n",
+ skb, hdr->opcode & ISCSI_OPCODE_MASK,
+ tcp_conn->in.datalen, hdr->itt);
+ offset += sizeof(struct cpl_iscsi_hdr_norss);
+@@ -613,8 +620,7 @@ int cxgb3i_conn_ulp2_xmit(struct iscsi_c
+ }
+
+ send_pdu:
+- err = cxgb3i_c3cn_send_pdus((struct s3_conn *)tcp_conn->sock,
+- skb, MSG_DONTWAIT | MSG_NOSIGNAL);
++ err = cxgb3i_c3cn_send_pdus((struct s3_conn *)tcp_conn->sock, skb);
+
+ if (err > 0) {
+ int pdulen = hdrlen + datalen + padlen;
+@@ -758,7 +764,8 @@ int cxgb3i_adapter_ulp_init(struct cxgb3
+ ddp = cxgb3i_alloc_big_mem(sizeof(struct cxgb3i_ddp_info) +
+ ppmax *
+ (sizeof(struct cxgb3i_gather_list *) +
+- sizeof(struct sk_buff *)));
++ sizeof(struct sk_buff *)),
++ GFP_KERNEL);
+ if (!ddp) {
+ cxgb3i_log_warn("snic %s unable to alloc ddp ppod 0x%u, "
+ "ddp disabled.\n", tdev->name, ppmax);
+--- a/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
++++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
+@@ -106,4 +106,27 @@ struct cpl_rx_data_ddp_norss {
+ void cxgb3i_conn_closing(struct s3_conn *);
+ void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn);
+ void cxgb3i_conn_tx_open(struct s3_conn *c3cn);
++
++/*
++ * large memory chunk allocation/release
++ * use vmalloc() if kmalloc() fails
++ */
++static inline void *cxgb3i_alloc_big_mem(unsigned int size, gfp_t gfp)
++{
++ void *p = kmalloc(size, gfp);
++
++ if (!p)
++ p = vmalloc(size);
++ if (p)
++ memset(p, 0, size);
++ return p;
++}
++
++static inline void cxgb3i_free_big_mem(void *addr)
++{
++ if (is_vmalloc_addr(addr))
++ vfree(addr);
++ else
++ kfree(addr);
++}
+ #endif
projects / people / pmueller / ipfire-2.x.git / blobdiff