+++ /dev/null
-Subject: add cxgb3i iscsi driver
-From: Karen Xie <kxie@chelsio.com>
-References: FATE#304154,bnc#433500
-
-Add Chelsio S3 iscsi initiator driver.
-This patch implements the cxgb3i iscsi connection offload.
-
-Signed-off-by: Karen Xie <kxie@chelsio.com>
-Signed-off-by: Chandra Seetharaman <sekharan@us.ibm.com>
-Signed-off-by: Hannes Reinecke <hare@suse.de>
----
-
----
- drivers/scsi/Kconfig | 2
- drivers/scsi/Makefile | 1
- drivers/scsi/cxgb3i/Kconfig | 7
- drivers/scsi/cxgb3i/Makefile | 5
- drivers/scsi/cxgb3i/cxgb3i.h | 179 +++
- drivers/scsi/cxgb3i/cxgb3i_init.c | 109 +
- drivers/scsi/cxgb3i/cxgb3i_iscsi.c | 854 ++++++++++++++
- drivers/scsi/cxgb3i/cxgb3i_offload.c | 2021 +++++++++++++++++++++++++++++++++++
- drivers/scsi/cxgb3i/cxgb3i_offload.h | 220 +++
- drivers/scsi/cxgb3i/cxgb3i_ulp2.c | 741 ++++++++++++
- drivers/scsi/cxgb3i/cxgb3i_ulp2.h | 108 +
- 11 files changed, 4247 insertions(+)
-
---- /dev/null
-+++ b/drivers/scsi/cxgb3i/cxgb3i.h
-@@ -0,0 +1,179 @@
-+/*
-+ * cxgb3i.h: Chelsio S3xx iSCSI driver.
-+ *
-+ * Copyright (c) 2008 Chelsio Communications, Inc.
-+ *
-+ * This program is free software; you can redistribute it and/or modify
-+ * it under the terms of the GNU General Public License as published by
-+ * the Free Software Foundation.
-+ *
-+ * Written by: Karen Xie (kxie@chelsio.com)
-+ */
-+
-+#ifndef __CXGB3I_H__
-+#define __CXGB3I_H__
-+
-+#include <linux/module.h>
-+#include <linux/moduleparam.h>
-+#include <linux/errno.h>
-+#include <linux/types.h>
-+#include <linux/list.h>
-+#include <linux/netdevice.h>
-+#include <linux/scatterlist.h>
-+
-+/* from cxgb3 LLD */
-+#include "common.h"
-+#include "t3_cpl.h"
-+#include "t3cdev.h"
-+#include "cxgb3_ctl_defs.h"
-+#include "cxgb3_offload.h"
-+#include "firmware_exports.h"
-+#include "cxgb3i_offload.h"
-+/* from iscsi */
-+#include "../iscsi_tcp.h"
-+
-+#define CXGB3I_SCSI_QDEPTH_DFLT 128
-+#define CXGB3I_MAX_TARGET CXGB3I_MAX_CONN
-+#define CXGB3I_MAX_LUN 512
-+#define ISCSI_PDU_HEADER_MAX (56 + 256) /* bhs + digests + ahs */
-+
-+struct cxgb3i_adapter;
-+struct cxgb3i_hba;
-+struct cxgb3i_endpoint;
-+
-+/**
-+ * struct cxgb3i_tag_format - cxgb3i ulp tag for steering pdu payload
-+ *
-+ * @idx_bits: # of bits used to store itt (from iscsi laryer)
-+ * @age_bits: # of bits used to store age (from iscsi laryer)
-+ * @rsvd_bits: # of bits used by h/w
-+ * @rsvd_shift: shift left
-+ * @rsvd_mask: bit mask
-+ * @rsvd_tag_mask: h/w tag bit mask
-+ *
-+ */
-+struct cxgb3i_tag_format {
-+ unsigned char idx_bits;
-+ unsigned char age_bits;
-+ unsigned char rsvd_bits;
-+ unsigned char rsvd_shift;
-+ u32 rsvd_mask;
-+ u32 rsvd_tag_mask;
-+};
-+
-+/**
-+ * struct cxgb3i_ddp_info - cxgb3i direct data placement for pdu payload
-+ *
-+ * @llimit: lower bound of the page pod memory
-+ * @ulimit: upper bound of the page pod memory
-+ * @nppods: # of page pod entries
-+ * @idx_last: page pod entry last used
-+ * @map_lock: lock to synchonize access to the page pod map
-+ * @map: page pod map
-+ */
-+struct cxgb3i_ddp_info {
-+ unsigned int llimit;
-+ unsigned int ulimit;
-+ unsigned int nppods;
-+ unsigned int idx_last;
-+ spinlock_t map_lock;
-+ u8 *map;
-+};
-+
-+/**
-+ * struct cxgb3i_hba - cxgb3i iscsi structure (per port)
-+ *
-+ * @snic: cxgb3i adapter containing this port
-+ * @ndev: pointer to netdev structure
-+ * @shost: pointer to scsi host structure
-+ */
-+struct cxgb3i_hba {
-+ struct cxgb3i_adapter *snic;
-+ struct net_device *ndev;
-+ struct Scsi_Host *shost;
-+};
-+
-+/**
-+ * struct cxgb3i_adapter - cxgb3i adapter structure (per pci)
-+ *
-+ * @listhead: list head to link elements
-+ * @lock: lock for this structure
-+ * @tdev: pointer to t3cdev used by cxgb3 driver
-+ * @pdev: pointer to pci dev
-+ * @hba_cnt: # of hbas (the same as # of ports)
-+ * @hba: all the hbas on this adapter
-+ * @tx_max_size: max. tx packet size supported
-+ * @rx_max_size: max. rx packet size supported
-+ * @tag_format: ulp tag format settings
-+ * @ddp: ulp ddp state
-+ */
-+struct cxgb3i_adapter {
-+ struct list_head list_head;
-+ spinlock_t lock;
-+ struct t3cdev *tdev;
-+ struct pci_dev *pdev;
-+ unsigned char hba_cnt;
-+ struct cxgb3i_hba *hba[MAX_NPORTS];
-+
-+ unsigned int tx_max_size;
-+ unsigned int rx_max_size;
-+
-+ struct cxgb3i_tag_format tag_format;
-+ struct cxgb3i_ddp_info ddp;
-+};
-+
-+/**
-+ * struct cxgb3i_conn - cxgb3i iscsi connection
-+ *
-+ * @tcp_conn: pointer to iscsi_tcp_conn structure
-+ * @listhead: list head to link elements
-+ * @conn: pointer to iscsi_conn structure
-+ * @hba: pointer to the hba this conn. is going through
-+ */
-+struct cxgb3i_conn {
-+ struct iscsi_tcp_conn tcp_conn;
-+ struct list_head list_head;
-+ struct cxgb3i_endpoint *cep;
-+ struct iscsi_conn *conn;
-+ struct cxgb3i_hba *hba;
-+};
-+
-+/**
-+ * struct cxgb3i_endpoint - iscsi tcp endpoint
-+ *
-+ * @c3cn: the h/w tcp connection representation
-+ * @hba: pointer to the hba this conn. is going through
-+ * @cconn: pointer to the associated cxgb3i iscsi connection
-+ */
-+struct cxgb3i_endpoint {
-+ struct s3_conn *c3cn;
-+ struct cxgb3i_hba *hba;
-+ struct cxgb3i_conn *cconn;
-+};
-+
-+/*
-+ * Function Prototypes
-+ */
-+int cxgb3i_iscsi_init(void);
-+void cxgb3i_iscsi_cleanup(void);
-+
-+struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *);
-+void cxgb3i_adapter_remove(struct t3cdev *);
-+int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *);
-+void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *);
-+
-+struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *);
-+struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *,
-+ struct net_device *);
-+void cxgb3i_hba_host_remove(struct cxgb3i_hba *);
-+
-+int cxgb3i_ulp2_init(void);
-+void cxgb3i_ulp2_cleanup(void);
-+int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *, int, int);
-+void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32,
-+ struct scatterlist *, unsigned int);
-+u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *, unsigned int,
-+ u32, unsigned int, struct scatterlist *,
-+ unsigned int);
-+int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *);
-+#endif
---- /dev/null
-+++ b/drivers/scsi/cxgb3i/cxgb3i_init.c
-@@ -0,0 +1,109 @@
-+/* cxgb3i_init.c: Chelsio S3xx iSCSI driver.
-+ *
-+ * Copyright (c) 2008 Chelsio Communications, Inc.
-+ *
-+ * This program is free software; you can redistribute it and/or modify
-+ * it under the terms of the GNU General Public License as published by
-+ * the Free Software Foundation.
-+ *
-+ * Written by: Karen Xie (kxie@chelsio.com)
-+ */
-+
-+#include "cxgb3i.h"
-+
-+#define DRV_MODULE_NAME "cxgb3i"
-+#define DRV_MODULE_VERSION "1.0.0"
-+#define DRV_MODULE_RELDATE "Jun. 1, 2008"
-+
-+static char version[] =
-+ "Chelsio S3xx iSCSI Driver " DRV_MODULE_NAME
-+ " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
-+
-+MODULE_AUTHOR("Karen Xie <kxie@chelsio.com>");
-+MODULE_DESCRIPTION("Chelsio S3xx iSCSI Driver");
-+MODULE_LICENSE("GPL");
-+MODULE_VERSION(DRV_MODULE_VERSION);
-+
-+static void open_s3_dev(struct t3cdev *);
-+static void close_s3_dev(struct t3cdev *);
-+
-+static cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS];
-+static struct cxgb3_client t3c_client = {
-+ .name = "iscsi_cxgb3",
-+ .handlers = cxgb3i_cpl_handlers,
-+ .add = open_s3_dev,
-+ .remove = close_s3_dev,
-+};
-+
-+/**
-+ * open_s3_dev - register with cxgb3 LLD
-+ * @t3dev: cxgb3 adapter instance
-+ */
-+static void open_s3_dev(struct t3cdev *t3dev)
-+{
-+ static int vers_printed;
-+
-+ if (!vers_printed) {
-+ printk(KERN_INFO "%s", version);
-+ vers_printed = 1;
-+ }
-+
-+ cxgb3i_log_debug("open cxgb3 %s.\n", t3dev->name);
-+ cxgb3i_sdev_add(t3dev, &t3c_client);
-+ cxgb3i_adapter_add(t3dev);
-+}
-+
-+/**
-+ * close_s3_dev - de-register with cxgb3 LLD
-+ * @t3dev: cxgb3 adapter instance
-+ */
-+static void close_s3_dev(struct t3cdev *t3dev)
-+{
-+ cxgb3i_log_debug("close cxgb3 %s.\n", t3dev->name);
-+ cxgb3i_adapter_remove(t3dev);
-+ cxgb3i_sdev_remove(t3dev);
-+}
-+
-+/**
-+ * cxgb3i_init_module - module init entry point
-+ *
-+ * initialize any driver wide global data structures and register itself
-+ * with the cxgb3 module
-+ */
-+static int __init cxgb3i_init_module(void)
-+{
-+ int err;
-+
-+ err = cxgb3i_sdev_init(cxgb3i_cpl_handlers);
-+ if (err < 0)
-+ return err;
-+
-+ err = cxgb3i_iscsi_init();
-+ if (err < 0)
-+ return err;
-+
-+ err = cxgb3i_ulp2_init();
-+ if (err < 0)
-+ return err;
-+
-+ cxgb3_register_client(&t3c_client);
-+
-+ return 0;
-+}
-+
-+/**
-+ * cxgb3i_exit_module - module cleanup/exit entry point
-+ *
-+ * go through the driver hba list and for each hba, release any resource held.
-+ * and unregisters iscsi transport and the cxgb3 module
-+ */
-+static void __exit cxgb3i_exit_module(void)
-+{
-+ cxgb3_unregister_client(&t3c_client);
-+ cxgb3i_ulp2_cleanup();
-+ cxgb3i_iscsi_cleanup();
-+ cxgb3i_sdev_cleanup();
-+}
-+
-+module_init(cxgb3i_init_module);
-+module_exit(cxgb3i_exit_module);
---- /dev/null
-+++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
-@@ -0,0 +1,854 @@
-+/* cxgb3i_iscsi.c: Chelsio S3xx iSCSI driver.
-+ *
-+ * Copyright (c) 2008 Chelsio Communications, Inc.
-+ *
-+ * This program is free software; you can redistribute it and/or modify
-+ * it under the terms of the GNU General Public License as published by
-+ * the Free Software Foundation.
-+ *
-+ * Written by: Karen Xie (kxie@chelsio.com)
-+ */
-+
-+#include <linux/inet.h>
-+#include <linux/crypto.h>
-+#include <net/tcp.h>
-+#include <scsi/scsi_cmnd.h>
-+#include <scsi/scsi_device.h>
-+#include <scsi/scsi_eh.h>
-+#include <scsi/scsi_host.h>
-+#include <scsi/scsi.h>
-+#include <scsi/iscsi_proto.h>
-+#include <scsi/libiscsi.h>
-+#include <scsi/scsi_transport_iscsi.h>
-+
-+#include "cxgb3i.h"
-+
-+static struct scsi_transport_template *cxgb3i_scsi_transport;
-+static struct scsi_host_template cxgb3i_host_template;
-+static struct iscsi_transport cxgb3i_iscsi_transport;
-+
-+static LIST_HEAD(cxgb3i_snic_list);
-+static DEFINE_RWLOCK(cxgb3i_snic_rwlock);
-+
-+/**
-+ * cxgb3i_adapter_add - init a s3 adapter structure and any h/w settings
-+ * @snic: pointer to adapter instance
-+ */
-+struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *t3dev)
-+{
-+ struct cxgb3i_adapter *snic;
-+ struct adapter *adapter = tdev2adap(t3dev);
-+ int i;
-+
-+ snic = kzalloc(sizeof(*snic), GFP_KERNEL);
-+ if (!snic) {
-+ cxgb3i_log_debug("cxgb3 %s, OOM.\n", t3dev->name);
-+ return NULL;
-+ }
-+
-+ spin_lock_init(&snic->lock);
-+ snic->tdev = t3dev;
-+ snic->pdev = adapter->pdev;
-+
-+ if (cxgb3i_adapter_ulp_init(snic))
-+ goto free_snic;
-+
-+ for_each_port(adapter, i) {
-+ snic->hba[i] = cxgb3i_hba_host_add(snic, adapter->port[i]);
-+ if (!snic->hba[i])
-+ goto ulp_cleanup;
-+ }
-+ snic->hba_cnt = adapter->params.nports;
-+
-+ /* add to the list */
-+ write_lock(&cxgb3i_snic_rwlock);
-+ list_add_tail(&snic->list_head, &cxgb3i_snic_list);
-+ write_unlock(&cxgb3i_snic_rwlock);
-+
-+ return snic;
-+
-+ulp_cleanup:
-+ cxgb3i_adapter_ulp_cleanup(snic);
-+free_snic:
-+ kfree(snic);
-+ return NULL;
-+}
-+
-+/**
-+ * cxgb3i_snic_cleanup - release all the resources held and cleanup h/w settings
-+ * @snic: pointer to adapter instance
-+ */
-+void cxgb3i_adapter_remove(struct t3cdev *t3dev)
-+{
-+ int i;
-+ struct cxgb3i_adapter *snic;
-+
-+ /* remove from the list */
-+ read_lock(&cxgb3i_snic_rwlock);
-+ list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
-+ if (snic->tdev == t3dev) {
-+ list_del(&snic->list_head);
-+ break;
-+ }
-+ }
-+ write_unlock(&cxgb3i_snic_rwlock);
-+
-+ if (snic) {
-+ for (i = 0; i < snic->hba_cnt; i++) {
-+ if (snic->hba[i]) {
-+ cxgb3i_hba_host_remove(snic->hba[i]);
-+ snic->hba[i] = NULL;
-+ }
-+ }
-+
-+ /* release ddp resources */
-+ cxgb3i_adapter_ulp_cleanup(snic);
-+ kfree(snic);
-+ }
-+}
-+
-+struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *ndev)
-+{
-+ struct cxgb3i_adapter *snic;
-+ int i;
-+
-+ read_lock(&cxgb3i_snic_rwlock);
-+ list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
-+ for (i = 0; i < snic->hba_cnt; i++) {
-+ if (snic->hba[i]->ndev == ndev) {
-+ read_unlock(&cxgb3i_snic_rwlock);
-+ return snic->hba[i];
-+ }
-+ }
-+ }
-+ read_unlock(&cxgb3i_snic_rwlock);
-+ return NULL;
-+}
-+
-+struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *snic,
-+ struct net_device *ndev)
-+{
-+ struct cxgb3i_hba *hba;
-+ struct Scsi_Host *shost;
-+ int err;
-+
-+ shost = iscsi_host_alloc(&cxgb3i_host_template,
-+ sizeof(struct cxgb3i_hba),
-+ CXGB3I_SCSI_QDEPTH_DFLT);
-+ if (!shost) {
-+ cxgb3i_log_info("iscsi_host_alloc failed.\n");
-+ return NULL;
-+ }
-+
-+ shost->transportt = cxgb3i_scsi_transport;
-+ shost->max_lun = CXGB3I_MAX_LUN;
-+ shost->max_id = CXGB3I_MAX_TARGET;
-+ shost->max_channel = 0;
-+ shost->max_cmd_len = 16;
-+
-+ hba = iscsi_host_priv(shost);
-+ hba->snic = snic;
-+ hba->ndev = ndev;
-+ hba->shost = shost;
-+
-+ pci_dev_get(snic->pdev);
-+ err = iscsi_host_add(shost, &snic->pdev->dev);
-+ if (err) {
-+ cxgb3i_log_info("iscsi_host_add failed.\n");
-+ goto pci_dev_put;
-+ }
-+
-+ cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n",
-+ shost, hba, shost->host_no);
-+
-+ return hba;
-+
-+pci_dev_put:
-+ pci_dev_put(snic->pdev);
-+ scsi_host_put(shost);
-+ return NULL;
-+}
-+
-+void cxgb3i_hba_host_remove(struct cxgb3i_hba *hba)
-+{
-+ cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n",
-+ hba->shost, hba, hba->shost->host_no);
-+ iscsi_host_remove(hba->shost);
-+ pci_dev_put(hba->snic->pdev);
-+ iscsi_host_free(hba->shost);
-+}
-+
-+/**
-+ * cxgb3i_ep_connect - establish TCP connection to target portal
-+ * @dst_addr: target IP address
-+ * @non_blocking: blocking or non-blocking call
-+ *
-+ * Initiates a TCP/IP connection to the dst_addr
-+ */
-+static struct iscsi_endpoint *cxgb3i_ep_connect(struct sockaddr *dst_addr,
-+ int non_blocking)
-+{
-+ struct iscsi_endpoint *ep;
-+ struct cxgb3i_endpoint *cep;
-+ struct cxgb3i_hba *hba;
-+ struct s3_conn *c3cn = NULL;
-+ int err = 0;
-+
-+ c3cn = cxgb3i_c3cn_create();
-+ if (!c3cn) {
-+ cxgb3i_log_info("ep connect OOM.\n");
-+ err = -ENOMEM;
-+ goto release_conn;
-+ }
-+
-+ err = cxgb3i_c3cn_connect(c3cn, (struct sockaddr_in *)dst_addr);
-+ if (err < 0) {
-+ cxgb3i_log_info("ep connect failed.\n");
-+ goto release_conn;
-+ }
-+ hba = cxgb3i_hba_find_by_netdev(c3cn->dst_cache->dev);
-+ if (!hba) {
-+ err = -ENOSPC;
-+ cxgb3i_log_info("NOT going through cxgbi device.\n");
-+ goto release_conn;
-+ }
-+ if (c3cn_in_state(c3cn, C3CN_STATE_CLOSE)) {
-+ err = -ENOSPC;
-+ cxgb3i_log_info("ep connect unable to connect.\n");
-+ goto release_conn;
-+ }
-+
-+ ep = iscsi_create_endpoint(sizeof(*cep));
-+ if (!ep) {
-+ err = -ENOMEM;
-+ cxgb3i_log_info("iscsi alloc ep, OOM.\n");
-+ goto release_conn;
-+ }
-+ cep = ep->dd_data;
-+ cep->c3cn = c3cn;
-+ cep->hba = hba;
-+
-+ cxgb3i_log_debug("ep 0x%p, 0x%p, c3cn 0x%p, hba 0x%p.\n",
-+ ep, cep, c3cn, hba);
-+ return ep;
-+
-+release_conn:
-+ cxgb3i_log_debug("conn 0x%p failed, release.\n", c3cn);
-+ if (c3cn)
-+ cxgb3i_c3cn_release(c3cn);
-+ return ERR_PTR(err);
-+}
-+
-+/**
-+ * cxgb3i_ep_poll - polls for TCP connection establishement
-+ * @ep: TCP connection (endpoint) handle
-+ * @timeout_ms: timeout value in milli secs
-+ *
-+ * polls for TCP connect request to complete
-+ */
-+static int cxgb3i_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
-+{
-+ struct cxgb3i_endpoint *cep = ep->dd_data;
-+ struct s3_conn *c3cn = cep->c3cn;
-+
-+ if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED))
-+ return 0;
-+ cxgb3i_log_debug("ep 0x%p, c3cn 0x%p established.\n", ep, c3cn);
-+ return 1;
-+}
-+
-+/**
-+ * cxgb3i_ep_disconnect - teardown TCP connection
-+ * @ep: TCP connection (endpoint) handle
-+ *
-+ * teardown TCP connection
-+ */
-+static void cxgb3i_ep_disconnect(struct iscsi_endpoint *ep)
-+{
-+ struct cxgb3i_endpoint *cep = ep->dd_data;
-+ struct cxgb3i_conn *cconn = cep->cconn;
-+
-+ cxgb3i_log_debug("ep 0x%p, cep 0x%p.\n", ep, cep);
-+
-+ if (cconn && cconn->conn) {
-+ struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
-+
-+ /*
-+ * stop the xmit path so the xmit_segment function is
-+ * not being called
-+ */
-+ write_lock_bh(&cep->c3cn->callback_lock);
-+ set_bit(ISCSI_SUSPEND_BIT, &cconn->conn->suspend_rx);
-+ cep->c3cn->user_data = NULL;
-+ cconn->cep = NULL;
-+ tcp_conn->sock = NULL;
-+ write_unlock_bh(&cep->c3cn->callback_lock);
-+ }
-+
-+ cxgb3i_log_debug("ep 0x%p, cep 0x%p, release c3cn 0x%p.\n",
-+ ep, cep, cep->c3cn);
-+ cxgb3i_c3cn_release(cep->c3cn);
-+ iscsi_destroy_endpoint(ep);
-+}
-+
-+/**
-+ * cxgb3i_session_create - create a new iscsi session
-+ * @cmds_max: max # of commands
-+ * @qdepth: scsi queue depth
-+ * @initial_cmdsn: initial iscsi CMDSN for this session
-+ * @host_no: pointer to return host no
-+ *
-+ * Creates a new iSCSI session
-+ */
-+static struct iscsi_cls_session *
-+cxgb3i_session_create(struct iscsi_endpoint *ep, u16 cmds_max, u16 qdepth,
-+ u32 initial_cmdsn, u32 *host_no)
-+{
-+ struct cxgb3i_endpoint *cep;
-+ struct cxgb3i_hba *hba;
-+ struct Scsi_Host *shost;
-+ struct iscsi_cls_session *cls_session;
-+ struct iscsi_session *session;
-+ int i;
-+
-+ if (!ep) {
-+ cxgb3i_log_error("%s, missing endpoint.\n", __func__);
-+ return NULL;
-+ }
-+
-+ cep = ep->dd_data;
-+ hba = cep->hba;
-+ shost = hba->shost;
-+ cxgb3i_log_debug("ep 0x%p, cep 0x%p, hba 0x%p.\n", ep, cep, hba);
-+ BUG_ON(hba != iscsi_host_priv(shost));
-+
-+ *host_no = shost->host_no;
-+
-+ cls_session = iscsi_session_setup(&cxgb3i_iscsi_transport, shost,
-+ cmds_max,
-+ sizeof(struct iscsi_tcp_task),
-+ initial_cmdsn, ISCSI_MAX_TARGET);
-+ if (!cls_session)
-+ return NULL;
-+ session = cls_session->dd_data;
-+
-+ for (i = 0; i < session->cmds_max; i++) {
-+ struct iscsi_task *task = session->cmds[i];
-+ struct iscsi_tcp_task *tcp_task = task->dd_data;
-+
-+ task->hdr = &tcp_task->hdr.cmd_hdr;
-+ task->hdr_max = sizeof(tcp_task->hdr) - ISCSI_DIGEST_SIZE;
-+ }
-+
-+ if (iscsi_r2tpool_alloc(session))
-+ goto remove_session;
-+
-+ return cls_session;
-+
-+remove_session:
-+ iscsi_session_teardown(cls_session);
-+ return NULL;
-+}
-+
-+/**
-+ * cxgb3i_session_destroy - destroys iscsi session
-+ * @cls_session: pointer to iscsi cls session
-+ *
-+ * Destroys an iSCSI session instance and releases its all resources held
-+ */
-+static void cxgb3i_session_destroy(struct iscsi_cls_session *cls_session)
-+{
-+ cxgb3i_log_debug("sess 0x%p.\n", cls_session);
-+ iscsi_r2tpool_free(cls_session->dd_data);
-+ iscsi_session_teardown(cls_session);
-+}
-+
-+/**
-+ * cxgb3i_conn_create - create iscsi connection instance
-+ * @cls_session: pointer to iscsi cls session
-+ * @cid: iscsi cid
-+ *
-+ * Creates a new iSCSI connection instance for a given session
-+ */
-+static inline void cxgb3i_conn_max_xmit_dlength(struct iscsi_conn *conn)
-+{
-+ struct cxgb3i_conn *cconn = conn->dd_data;
-+
-+ if (conn->max_xmit_dlength)
-+ conn->max_xmit_dlength = min_t(unsigned int,
-+ conn->max_xmit_dlength,
-+ cconn->hba->snic->tx_max_size -
-+ ISCSI_PDU_HEADER_MAX);
-+ else
-+ conn->max_xmit_dlength = cconn->hba->snic->tx_max_size -
-+ ISCSI_PDU_HEADER_MAX;
-+ cxgb3i_log_debug("conn 0x%p, max xmit %u.\n",
-+ conn, conn->max_xmit_dlength);
-+}
-+
-+static inline void cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
-+{
-+ struct cxgb3i_conn *cconn = conn->dd_data;
-+
-+ if (conn->max_recv_dlength)
-+ conn->max_recv_dlength = min_t(unsigned int,
-+ conn->max_recv_dlength,
-+ cconn->hba->snic->rx_max_size -
-+ ISCSI_PDU_HEADER_MAX);
-+ else
-+ conn->max_recv_dlength = cconn->hba->snic->rx_max_size -
-+ ISCSI_PDU_HEADER_MAX;
-+ cxgb3i_log_debug("conn 0x%p, max recv %u.\n",
-+ conn, conn->max_recv_dlength);
-+}
-+
-+static struct iscsi_cls_conn *cxgb3i_conn_create(struct iscsi_cls_session
-+ *cls_session, u32 cid)
-+{
-+ struct iscsi_cls_conn *cls_conn;
-+ struct iscsi_conn *conn;
-+ struct cxgb3i_conn *cconn;
-+
-+ cxgb3i_log_debug("sess 0x%p, cid %u.\n", cls_session, cid);
-+
-+ cls_conn = iscsi_conn_setup(cls_session, sizeof(*cconn), cid);
-+ if (!cls_conn)
-+ return NULL;
-+ conn = cls_conn->dd_data;
-+
-+ cconn = conn->dd_data;
-+ cconn->tcp_conn.iscsi_conn = conn;
-+ cconn->conn = conn;
-+
-+ return cls_conn;
-+}
-+
-+/**
-+ * cxgb3i_conn_xmit_segment - transmit segment
-+ * @conn: pointer to iscsi conn
-+ */
-+static int cxgb3i_conn_xmit_segment(struct iscsi_conn *conn)
-+{
-+ struct cxgb3i_conn *cconn = conn->dd_data;
-+ struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
-+ struct iscsi_segment *segment = &tcp_conn->out.segment;
-+
-+ if (segment->total_copied < segment->total_size)
-+ return cxgb3i_conn_ulp2_xmit(conn);
-+ return 0;
-+}
-+
-+/**
-+ * cxgb3i_conn_bind - binds iscsi sess, conn and endpoint together
-+ * @cls_session: pointer to iscsi cls session
-+ * @cls_conn: pointer to iscsi cls conn
-+ * @transport_eph: 64-bit EP handle
-+ * @is_leading: leading connection on this session?
-+ *
-+ * Binds together an iSCSI session, an iSCSI connection and a
-+ * TCP connection. This routine returns error code if the TCP
-+ * connection does not belong on the device iSCSI sess/conn is bound
-+ */
-+
-+static int cxgb3i_conn_bind(struct iscsi_cls_session *cls_session,
-+ struct iscsi_cls_conn *cls_conn,
-+ u64 transport_eph, int is_leading)
-+{
-+ struct iscsi_conn *conn = cls_conn->dd_data;
-+ struct cxgb3i_conn *cconn = conn->dd_data;
-+ struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
-+ struct iscsi_endpoint *ep;
-+ struct cxgb3i_endpoint *cep;
-+ struct s3_conn *c3cn;
-+ int err;
-+
-+ ep = iscsi_lookup_endpoint(transport_eph);
-+ if (!ep)
-+ return -EINVAL;
-+
-+ cxgb3i_log_debug("ep 0x%p, cls sess 0x%p, cls conn 0x%p.\n",
-+ ep, cls_session, cls_conn);
-+
-+ err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
-+ if (err)
-+ return -EINVAL;
-+
-+ cep = ep->dd_data;
-+ c3cn = cep->c3cn;
-+
-+ read_lock(&c3cn->callback_lock);
-+ /* mnc: TODO don't abuse iscsi_tcp fields */
-+ tcp_conn->sock = (struct socket *)c3cn;
-+ c3cn->user_data = conn;
-+ read_unlock(&c3cn->callback_lock);
-+
-+ cconn->hba = cep->hba;
-+ cconn->cep = cep;
-+ cep->cconn = cconn;
-+
-+ cxgb3i_conn_max_xmit_dlength(conn);
-+ cxgb3i_conn_max_recv_dlength(conn);
-+
-+ spin_lock_bh(&conn->session->lock);
-+ sprintf(conn->portal_address, NIPQUAD_FMT,
-+ NIPQUAD(c3cn->daddr.sin_addr.s_addr));
-+ conn->portal_port = ntohs(c3cn->daddr.sin_port);
-+ spin_unlock_bh(&conn->session->lock);
-+
-+ tcp_conn->xmit_segment = cxgb3i_conn_xmit_segment;
-+ iscsi_tcp_hdr_recv_prep(tcp_conn);
-+
-+ return 0;
-+}
-+
-+/**
-+ * cxgb3i_conn_get_param - return iscsi connection parameter to caller
-+ * @cls_conn: pointer to iscsi cls conn
-+ * @param: parameter type identifier
-+ * @buf: buffer pointer
-+ *
-+ * returns iSCSI connection parameters
-+ */
-+static int cxgb3i_conn_get_param(struct iscsi_cls_conn *cls_conn,
-+ enum iscsi_param param, char *buf)
-+{
-+ struct iscsi_conn *conn = cls_conn->dd_data;
-+ int len;
-+
-+ cxgb3i_log_debug("cls_conn 0x%p, param %d.\n", cls_conn, param);
-+
-+ switch (param) {
-+ case ISCSI_PARAM_CONN_PORT:
-+ spin_lock_bh(&conn->session->lock);
-+ len = sprintf(buf, "%hu\n", conn->portal_port);
-+ spin_unlock_bh(&conn->session->lock);
-+ break;
-+ case ISCSI_PARAM_CONN_ADDRESS:
-+ spin_lock_bh(&conn->session->lock);
-+ len = sprintf(buf, "%s\n", conn->portal_address);
-+ spin_unlock_bh(&conn->session->lock);
-+ break;
-+ default:
-+ return iscsi_conn_get_param(cls_conn, param, buf);
-+ }
-+
-+ return len;
-+}
-+
-+static int cxgb3i_conn_set_param(struct iscsi_cls_conn *cls_conn,
-+ enum iscsi_param param, char *buf, int buflen)
-+{
-+ struct iscsi_conn *conn = cls_conn->dd_data;
-+ struct iscsi_session *session = conn->session;
-+ struct cxgb3i_conn *cconn = conn->dd_data;
-+ int value, err = 0;
-+
-+ switch (param) {
-+ case ISCSI_PARAM_HDRDGST_EN:
-+ err = iscsi_set_param(cls_conn, param, buf, buflen);
-+ if (!err && conn->hdrdgst_en)
-+ cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en,
-+ conn->datadgst_en);
-+ break;
-+ case ISCSI_PARAM_DATADGST_EN:
-+ err = iscsi_set_param(cls_conn, param, buf, buflen);
-+ if (!err && conn->datadgst_en)
-+ cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en,
-+ conn->datadgst_en);
-+ break;
-+ case ISCSI_PARAM_MAX_R2T:
-+ sscanf(buf, "%d", &value);
-+ if (value <= 0 || !is_power_of_2(value))
-+ return -EINVAL;
-+ if (session->max_r2t == value)
-+ break;
-+ iscsi_r2tpool_free(session);
-+ err = iscsi_set_param(cls_conn, param, buf, buflen);
-+ if (!err && iscsi_r2tpool_alloc(session))
-+ return -ENOMEM;
-+ case ISCSI_PARAM_MAX_RECV_DLENGTH:
-+ err = iscsi_set_param(cls_conn, param, buf, buflen);
-+ cxgb3i_conn_max_recv_dlength(conn);
-+ break;
-+ case ISCSI_PARAM_MAX_XMIT_DLENGTH:
-+ err = iscsi_set_param(cls_conn, param, buf, buflen);
-+ cxgb3i_conn_max_xmit_dlength(conn);
-+ break;
-+ default:
-+ return iscsi_set_param(cls_conn, param, buf, buflen);
-+ }
-+ return err;
-+}
-+
-+/**
-+ * cxgb3i_host_set_param - configure host (adapter) related parameters
-+ * @shost: scsi host pointer
-+ * @param: parameter type identifier
-+ * @buf: buffer pointer
-+ */
-+static int cxgb3i_host_set_param(struct Scsi_Host *shost,
-+ enum iscsi_host_param param,
-+ char *buf, int buflen)
-+{
-+ struct cxgb3i_hba *hba = iscsi_host_priv(shost);
-+
-+ cxgb3i_log_debug("param %d, buf %s.\n", param, buf);
-+
-+ if (hba && param == ISCSI_HOST_PARAM_IPADDRESS) {
-+ __be32 addr = in_aton(buf);
-+ cxgb3i_set_private_ipv4addr(hba->ndev, addr);
-+ return 0;
-+ }
-+
-+ return iscsi_host_get_param(shost, param, buf);
-+}
-+
-+/**
-+ * cxgb3i_host_get_param - returns host (adapter) related parameters
-+ * @shost: scsi host pointer
-+ * @param: parameter type identifier
-+ * @buf: buffer pointer
-+ */
-+static int cxgb3i_host_get_param(struct Scsi_Host *shost,
-+ enum iscsi_host_param param, char *buf)
-+{
-+ struct cxgb3i_hba *hba = iscsi_host_priv(shost);
-+ int i;
-+ int len = 0;
-+
-+ cxgb3i_log_debug("hba %s, param %d.\n", hba->ndev->name, param);
-+
-+ switch (param) {
-+ case ISCSI_HOST_PARAM_HWADDRESS:
-+ for (i = 0; i < 6; i++)
-+ len +=
-+ sprintf(buf + len, "%02x.",
-+ hba->ndev->dev_addr[i]);
-+ len--;
-+ buf[len] = '\0';
-+ break;
-+ case ISCSI_HOST_PARAM_NETDEV_NAME:
-+ len = sprintf(buf, "%s\n", hba->ndev->name);
-+ break;
-+ case ISCSI_HOST_PARAM_IPADDRESS:
-+ {
-+ __be32 addr;
-+
-+ addr = cxgb3i_get_private_ipv4addr(hba->ndev);
-+ len = sprintf(buf, "%u.%u.%u.%u", NIPQUAD(addr));
-+ break;
-+ }
-+ default:
-+ return iscsi_host_get_param(shost, param, buf);
-+ }
-+ return len;
-+}
-+
-+/**
-+ * cxgb3i_conn_get_stats - returns iSCSI stats
-+ * @cls_conn: pointer to iscsi cls conn
-+ * @stats: pointer to iscsi statistic struct
-+ */
-+static void cxgb3i_conn_get_stats(struct iscsi_cls_conn *cls_conn,
-+ struct iscsi_stats *stats)
-+{
-+ struct iscsi_conn *conn = cls_conn->dd_data;
-+
-+ stats->txdata_octets = conn->txdata_octets;
-+ stats->rxdata_octets = conn->rxdata_octets;
-+ stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
-+ stats->dataout_pdus = conn->dataout_pdus_cnt;
-+ stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
-+ stats->datain_pdus = conn->datain_pdus_cnt;
-+ stats->r2t_pdus = conn->r2t_pdus_cnt;
-+ stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
-+ stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
-+ stats->digest_err = 0;
-+ stats->timeout_err = 0;
-+ stats->custom_length = 1;
-+ strcpy(stats->custom[0].desc, "eh_abort_cnt");
-+ stats->custom[0].value = conn->eh_abort_cnt;
-+}
-+
-+static inline u32 tag_base(struct cxgb3i_tag_format *format,
-+ unsigned int idx, unsigned int age)
-+{
-+ u32 sw_bits = idx | (age << format->idx_bits);
-+ u32 tag = sw_bits >> format->rsvd_shift;
-+
-+ tag <<= format->rsvd_bits + format->rsvd_shift;
-+ tag |= sw_bits & ((1 << format->rsvd_shift) - 1);
-+ return tag;
-+}
-+
-+static inline void cxgb3i_parse_tag(struct cxgb3i_tag_format *format,
-+ u32 tag, u32 *rsvd_bits, u32 *sw_bits)
-+{
-+ if (rsvd_bits)
-+ *rsvd_bits = (tag >> format->rsvd_shift) & format->rsvd_mask;
-+ if (sw_bits) {
-+ *sw_bits = (tag >> (format->rsvd_shift + format->rsvd_bits))
-+ << format->rsvd_shift;
-+ *sw_bits |= tag & ((1 << format->rsvd_shift) - 1);
-+ }
-+}
-+
-+
-+static void cxgb3i_parse_itt(struct iscsi_conn *conn, itt_t itt,
-+ int *idx, int *age)
-+{
-+ struct cxgb3i_conn *cconn = conn->dd_data;
-+ struct cxgb3i_adapter *snic = cconn->hba->snic;
-+ u32 sw_bits;
-+
-+ cxgb3i_parse_tag(&snic->tag_format, itt, NULL, &sw_bits);
-+ if (idx)
-+ *idx = sw_bits & ISCSI_ITT_MASK;
-+ if (age)
-+ *age = (sw_bits >> snic->tag_format.idx_bits) & ISCSI_AGE_MASK;
-+}
-+
-+static int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
-+{
-+ struct scsi_cmnd *sc = task->sc;
-+ struct iscsi_conn *conn = task->conn;
-+ struct iscsi_session *sess = conn->session;
-+ struct cxgb3i_conn *cconn = conn->dd_data;
-+ struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
-+ struct cxgb3i_adapter *snic = cconn->hba->snic;
-+ u32 sw_tag = tag_base(&snic->tag_format, task->itt, sess->age);
-+ u32 tag = RESERVED_ITT;
-+
-+ if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE)) {
-+ struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
-+ tag =
-+ cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag,
-+ scsi_out(sc)->length,
-+ scsi_out(sc)->table.sgl,
-+ scsi_out(sc)->table.nents);
-+ }
-+ if (tag == RESERVED_ITT)
-+ tag = sw_tag | (snic->tag_format.rsvd_mask <<
-+ snic->tag_format.rsvd_shift);
-+ *hdr_itt = htonl(tag);
-+ return 0;
-+}
-+
-+static void cxgb3i_release_itt(struct iscsi_task *task, itt_t hdr_itt)
-+{
-+ struct scsi_cmnd *sc = task->sc;
-+ struct iscsi_conn *conn = task->conn;
-+ struct cxgb3i_conn *cconn = conn->dd_data;
-+ struct cxgb3i_adapter *snic = cconn->hba->snic;
-+
-+ hdr_itt = ntohl(hdr_itt);
-+ if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE))
-+ cxgb3i_ddp_tag_release(snic, hdr_itt,
-+ scsi_out(sc)->table.sgl,
-+ scsi_out(sc)->table.nents);
-+}
-+
-+/**
-+ * cxgb3i_host_template -- Scsi_Host_Template structure
-+ * used when registering with the scsi mid layer
-+ */
-+static struct scsi_host_template cxgb3i_host_template = {
-+ .module = THIS_MODULE,
-+ .name = "Chelsio S3xx iSCSI Initiator",
-+ .proc_name = "cxgb3i",
-+ .queuecommand = iscsi_queuecommand,
-+ .change_queue_depth = iscsi_change_queue_depth,
-+ .can_queue = 128 * (ISCSI_DEF_XMIT_CMDS_MAX - 1),
-+ .sg_tablesize = SG_ALL,
-+ .max_sectors = 0xFFFF,
-+ .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
-+ .eh_abort_handler = iscsi_eh_abort,
-+ .eh_device_reset_handler = iscsi_eh_device_reset,
-+ .eh_target_reset_handler = iscsi_eh_host_reset,
-+ .use_clustering = DISABLE_CLUSTERING,
-+ .this_id = -1,
-+};
-+
-+static struct iscsi_transport cxgb3i_iscsi_transport = {
-+ .owner = THIS_MODULE,
-+ .name = "cxgb3i",
-+ .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
-+ | CAP_DATADGST | CAP_DIGEST_OFFLOAD,
-+ .param_mask = ISCSI_MAX_RECV_DLENGTH |
-+ ISCSI_MAX_XMIT_DLENGTH |
-+ ISCSI_HDRDGST_EN |
-+ ISCSI_DATADGST_EN |
-+ ISCSI_INITIAL_R2T_EN |
-+ ISCSI_MAX_R2T |
-+ ISCSI_IMM_DATA_EN |
-+ ISCSI_FIRST_BURST |
-+ ISCSI_MAX_BURST |
-+ ISCSI_PDU_INORDER_EN |
-+ ISCSI_DATASEQ_INORDER_EN |
-+ ISCSI_ERL |
-+ ISCSI_CONN_PORT |
-+ ISCSI_CONN_ADDRESS |
-+ ISCSI_EXP_STATSN |
-+ ISCSI_PERSISTENT_PORT |
-+ ISCSI_PERSISTENT_ADDRESS |
-+ ISCSI_TARGET_NAME | ISCSI_TPGT |
-+ ISCSI_USERNAME | ISCSI_PASSWORD |
-+ ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
-+ ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
-+ ISCSI_LU_RESET_TMO |
-+ ISCSI_PING_TMO | ISCSI_RECV_TMO |
-+ ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
-+ .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
-+ ISCSI_HOST_INITIATOR_NAME | ISCSI_HOST_NETDEV_NAME,
-+ .get_host_param = cxgb3i_host_get_param,
-+ .set_host_param = cxgb3i_host_set_param,
-+ /* session management */
-+ .create_session = cxgb3i_session_create,
-+ .destroy_session = cxgb3i_session_destroy,
-+ .get_session_param = iscsi_session_get_param,
-+ /* connection management */
-+ .create_conn = cxgb3i_conn_create,
-+ .bind_conn = cxgb3i_conn_bind,
-+ .destroy_conn = iscsi_conn_teardown,
-+ .start_conn = iscsi_conn_start,
-+ .stop_conn = iscsi_conn_stop,
-+ .get_conn_param = cxgb3i_conn_get_param,
-+ .set_param = cxgb3i_conn_set_param,
-+ .get_stats = cxgb3i_conn_get_stats,
-+ /* pdu xmit req. from user space */
-+ .send_pdu = iscsi_conn_send_pdu,
-+ /* task */
-+ .init_task = iscsi_tcp_task_init,
-+ .xmit_task = iscsi_tcp_task_xmit,
-+ .cleanup_task = iscsi_tcp_cleanup_task,
-+ .parse_itt = cxgb3i_parse_itt,
-+ .reserve_itt = cxgb3i_reserve_itt,
-+ .release_itt = cxgb3i_release_itt,
-+ /* TCP connect/disconnect */
-+ .ep_connect = cxgb3i_ep_connect,
-+ .ep_poll = cxgb3i_ep_poll,
-+ .ep_disconnect = cxgb3i_ep_disconnect,
-+ /* Error recovery timeout call */
-+ .session_recovery_timedout = iscsi_session_recovery_timedout,
-+};
-+
-+int cxgb3i_iscsi_init(void)
-+{
-+ cxgb3i_scsi_transport =
-+ iscsi_register_transport(&cxgb3i_iscsi_transport);
-+ if (!cxgb3i_scsi_transport) {
-+ cxgb3i_log_error("Could not register cxgb3i transport.\n");
-+ return -ENODEV;
-+ }
-+ cxgb3i_log_debug("cxgb3i transport 0x%p.\n", cxgb3i_scsi_transport);
-+ return 0;
-+}
-+
-+void cxgb3i_iscsi_cleanup(void)
-+{
-+ if (cxgb3i_scsi_transport) {
-+ cxgb3i_log_debug("cxgb3i transport 0x%p.\n",
-+ cxgb3i_scsi_transport);
-+ iscsi_unregister_transport(&cxgb3i_iscsi_transport);
-+ }
-+}
---- /dev/null
-+++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
-@@ -0,0 +1,2021 @@
-+/*
-+ * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
-+ *
-+ * Written by Dimitris Michailidis (dm@chelsio.com)
-+ *
-+ * This program is distributed in the hope that it will be useful, but WITHOUT
-+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-+ * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
-+ * release for licensing terms and conditions.
-+ */
-+
-+#include <linux/if_vlan.h>
-+#include <linux/version.h>
-+
-+#include "cxgb3_defs.h"
-+#include "cxgb3_ctl_defs.h"
-+#include "firmware_exports.h"
-+#include "cxgb3i_offload.h"
-+#include "cxgb3i_ulp2.h"
-+
-+static int cxgb3_rcv_win = 256 * 1024;
-+module_param(cxgb3_rcv_win, int, 0644);
-+MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
-+
-+static int cxgb3_snd_win = 64 * 1024;
-+module_param(cxgb3_snd_win, int, 0644);
-+MODULE_PARM_DESC(cxgb3_snd_win, "TCP send window in bytes (default=64KB)");
-+
-+static int cxgb3_rx_credit_thres = 10 * 1024;
-+module_param(cxgb3_rx_credit_thres, int, 0644);
-+MODULE_PARM_DESC(rx_credit_thres,
-+ "RX credits return threshold in bytes (default=10KB)");
-+
-+static unsigned int cxgb3_max_connect = 8 * 1024;
-+module_param(cxgb3_max_connect, uint, 0644);
-+MODULE_PARM_DESC(cxgb3_max_connect, "Max. # of connections (default=8092)");
-+
-+static unsigned int cxgb3_sport_base = 20000;
-+module_param(cxgb3_sport_base, uint, 0644);
-+MODULE_PARM_DESC(cxgb3_sport_base, "starting port number (default=20000)");
-+
-+#ifdef __DEBUG_C3CN_TX__
-+#define c3cn_tx_debug cxgb3i_log_debug
-+#else
-+#define c3cn_tx_debug(fmt...)
-+#endif
-+
-+/* connection flags */
-+static inline void c3cn_set_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
-+{
-+ __set_bit(flag, &c3cn->flags);
-+ c3cn_conn_debug("c3cn 0x%p, set %d, s 0x%x, f 0x%lx.\n",
-+ c3cn, flag, c3cn->state, c3cn->flags);
-+}
-+
-+static inline void c3cn_reset_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
-+{
-+ __clear_bit(flag, &c3cn->flags);
-+ c3cn_conn_debug("c3cn 0x%p, clear %d, s 0x%x, f 0x%lx.\n",
-+ c3cn, flag, c3cn->state, c3cn->flags);
-+}
-+
-+static inline int c3cn_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
-+{
-+ if (c3cn == NULL)
-+ return 0;
-+ return test_bit(flag, &c3cn->flags);
-+}
-+
-+/* connection state */
-+static void c3cn_set_state(struct s3_conn *c3cn, int state)
-+{
-+ c3cn_conn_debug("c3cn 0x%p state -> 0x%x.\n", c3cn, state);
-+ c3cn->state = state;
-+}
-+
-+/* connection reference count */
-+static inline void c3cn_hold(struct s3_conn *c3cn)
-+{
-+ atomic_inc(&c3cn->refcnt);
-+}
-+
-+static inline void c3cn_put(struct s3_conn *c3cn)
-+{
-+ if (atomic_dec_and_test(&c3cn->refcnt)) {
-+ c3cn_conn_debug("free c3cn 0x%p, 0x%x, 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+ kfree(c3cn);
-+ }
-+}
-+
-+/* minimal port allocation management scheme */
-+static spinlock_t sport_map_lock;
-+static unsigned int sport_map_next;
-+static unsigned long *sport_map;
-+
-+/*
-+ * Find a free source port in our allocation map. We use a very simple rotor
-+ * scheme to look for the next free port.
-+ *
-+ * If a source port has been specified make sure that it doesn't collide with
-+ * our normal source port allocation map. If it's outside the range of our
-+ * allocation scheme just let them use it.
-+ */
-+static int c3cn_get_port(struct s3_conn *c3cn)
-+{
-+ unsigned int start;
-+
-+ if (!sport_map)
-+ goto error_out;
-+
-+ if (c3cn->saddr.sin_port != 0) {
-+ int sport = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
-+ int err = 0;
-+
-+ if (sport < 0 || sport >= cxgb3_max_connect)
-+ return 0;
-+ spin_lock(&sport_map_lock);
-+ err = __test_and_set_bit(sport, sport_map);
-+ spin_unlock(&sport_map_lock);
-+ return err ? -EADDRINUSE : 0;
-+ }
-+
-+ spin_lock(&sport_map_lock);
-+ start = sport_map_next;
-+ do {
-+ unsigned int new = sport_map_next;
-+ if (++sport_map_next >= cxgb3_max_connect)
-+ sport_map_next = 0;
-+ if (!(__test_and_set_bit(new, sport_map))) {
-+ spin_unlock(&sport_map_lock);
-+ c3cn_conn_debug("reserve port %u.\n",
-+ cxgb3_sport_base + new);
-+ c3cn->saddr.sin_port = htons(cxgb3_sport_base + new);
-+ return 0;
-+ }
-+ } while (sport_map_next != start);
-+ spin_unlock(&sport_map_lock);
-+
-+error_out:
-+ return -EADDRNOTAVAIL;
-+}
-+
-+/*
-+ * Deallocate a source port from the allocation map. If the source port is
-+ * outside our allocation range just return -- the caller is responsible for
-+ * keeping track of their port usage outside of our allocation map.
-+ */
-+static void c3cn_put_port(struct s3_conn *c3cn)
-+{
-+ if (c3cn->saddr.sin_port) {
-+ int old = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
-+ c3cn->saddr.sin_port = 0;
-+
-+ if (old < 0 || old >= cxgb3_max_connect)
-+ return;
-+
-+ c3cn_conn_debug("release port %u.\n", cxgb3_sport_base + old);
-+ spin_lock(&sport_map_lock);
-+ __clear_bit(old, sport_map);
-+ spin_unlock(&sport_map_lock);
-+ }
-+}
-+
-+static void c3cn_reset_timer(struct s3_conn *c3cn, struct timer_list *timer,
-+ unsigned long expires)
-+{
-+ if (!mod_timer(timer, expires))
-+ c3cn_hold(c3cn);
-+}
-+
-+typedef int (cxgb3_cpl_handler_decl) (struct t3cdev *,
-+ struct sk_buff *, void *);
-+
-+static cxgb3_cpl_handler_decl do_act_establish;
-+static cxgb3_cpl_handler_decl do_act_open_rpl;
-+static cxgb3_cpl_handler_decl do_wr_ack;
-+static cxgb3_cpl_handler_decl do_peer_close;
-+static cxgb3_cpl_handler_decl do_abort_req;
-+static cxgb3_cpl_handler_decl do_abort_rpl;
-+static cxgb3_cpl_handler_decl do_close_con_rpl;
-+static cxgb3_cpl_handler_decl do_iscsi_hdr;
-+
-+static LIST_HEAD(cxgb3_list);
-+static DEFINE_MUTEX(cxgb3_list_lock);
-+
-+/*
-+ * For ULP connections HW may inserts digest bytes into the pdu. This array
-+ * contains the compensating extra lengths for ULP packets. It is indexed by
-+ * a packet's ULP submode.
-+ */
-+static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 };
-+
-+/*
-+ * Return the length of any HW additions that will be made to a Tx packet.
-+ * Such additions can happen for some types of ULP packets.
-+ */
-+static inline unsigned int ulp_extra_len(const struct sk_buff *skb)
-+{
-+ return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3];
-+}
-+
-+/*
-+ * Size of WRs in bytes. Note that we assume all devices we are handling have
-+ * the same WR size.
-+ */
-+static unsigned int wrlen __read_mostly;
-+
-+/*
-+ * The number of WRs needed for an skb depends on the number of page fragments
-+ * in the skb and whether it has any payload in its main body. This maps the
-+ * length of the gather list represented by an skb into the # of necessary WRs.
-+ */
-+static unsigned int skb_wrs[MAX_SKB_FRAGS + 2] __read_mostly;
-+
-+static void s3_init_wr_tab(unsigned int wr_len)
-+{
-+ int i;
-+
-+ if (skb_wrs[1]) /* already initialized */
-+ return;
-+
-+ for (i = 1; i < ARRAY_SIZE(skb_wrs); i++) {
-+ int sgl_len = (3 * i) / 2 + (i & 1);
-+
-+ sgl_len += 3;
-+ skb_wrs[i] = (sgl_len <= wr_len
-+ ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
-+ }
-+
-+ wrlen = wr_len * 8;
-+}
-+
-+/*
-+ * cxgb3i API operations.
-+ */
-+/*
-+ * large memory chunk allocation/release
-+ */
-+void *cxgb3i_alloc_big_mem(unsigned int size)
-+{
-+ void *p = kmalloc(size, GFP_KERNEL);
-+ if (!p)
-+ p = vmalloc(size);
-+ if (p)
-+ memset(p, 0, size);
-+ return p;
-+}
-+
-+void cxgb3i_free_big_mem(void *addr)
-+{
-+ if (is_vmalloc_addr(addr))
-+ vfree(addr);
-+ else
-+ kfree(addr);
-+}
-+
-+void cxgb3i_sdev_cleanup(void)
-+{
-+ if (sport_map)
-+ cxgb3i_free_big_mem(sport_map);
-+}
-+
-+int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers)
-+{
-+ cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish;
-+ cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl;
-+ cpl_handlers[CPL_PEER_CLOSE] = do_peer_close;
-+ cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req;
-+ cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl;
-+ cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl;
-+ cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack;
-+ cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr;
-+
-+ if (cxgb3_max_connect > CXGB3I_MAX_CONN)
-+ cxgb3_max_connect = CXGB3I_MAX_CONN;
-+ sport_map = cxgb3i_alloc_big_mem(DIV_ROUND_UP(cxgb3_max_connect,
-+ 8 *
-+ sizeof(unsigned long)));
-+ if (!sport_map)
-+ return -ENOMEM;
-+ return 0;
-+}
-+
-+void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client)
-+{
-+ struct cxgb3i_sdev_data *cdata;
-+ struct adap_ports *ports;
-+ struct ofld_page_info rx_page_info;
-+ unsigned int wr_len;
-+ int i;
-+
-+ cdata = kzalloc(sizeof *cdata, GFP_KERNEL);
-+ if (!cdata)
-+ return;
-+ ports = kzalloc(sizeof *ports, GFP_KERNEL);
-+ if (!ports)
-+ goto free_ports;
-+ cdata->ports = ports;
-+
-+ if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 ||
-+ cdev->ctl(cdev, GET_PORTS, cdata->ports) < 0 ||
-+ cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0)
-+ goto free_ports;
-+
-+ s3_init_wr_tab(wr_len);
-+
-+ INIT_LIST_HEAD(&cdata->list);
-+ cdata->cdev = cdev;
-+ cdata->client = client;
-+ cdata->rx_page_size = rx_page_info.page_size;
-+ skb_queue_head_init(&cdata->deferq);
-+
-+ for (i = 0; i < ports->nports; i++)
-+ NDEV2CDATA(ports->lldevs[i]) = cdata;
-+
-+ mutex_lock(&cxgb3_list_lock);
-+ list_add_tail(&cdata->list, &cxgb3_list);
-+ mutex_unlock(&cxgb3_list_lock);
-+
-+ return;
-+
-+free_ports:
-+ kfree(ports);
-+ kfree(cdata);
-+}
-+
-+void cxgb3i_sdev_remove(struct t3cdev *cdev)
-+{
-+ struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
-+ struct adap_ports *ports = cdata->ports;
-+ int i;
-+
-+ for (i = 0; i < ports->nports; i++)
-+ NDEV2CDATA(ports->lldevs[i]) = NULL;
-+
-+ mutex_lock(&cxgb3_list_lock);
-+ list_del(&cdata->list);
-+ mutex_unlock(&cxgb3_list_lock);
-+
-+ kfree(ports);
-+ kfree(cdata);
-+}
-+
-+/*
-+ * Return TRUE if the specified net device is for a port on one of our
-+ * registered adapters.
-+ */
-+static int is_cxgb3_dev(struct net_device *dev)
-+{
-+ struct cxgb3i_sdev_data *cdata;
-+
-+ mutex_lock(&cxgb3_list_lock);
-+ list_for_each_entry(cdata, &cxgb3_list, list) {
-+ struct adap_ports *ports = cdata->ports;
-+ int i;
-+
-+ for (i = 0; i < ports->nports; i++)
-+ if (dev == ports->lldevs[i]) {
-+ mutex_unlock(&cxgb3_list_lock);
-+ return 1;
-+ }
-+ }
-+ mutex_unlock(&cxgb3_list_lock);
-+ return 0;
-+}
-+
-+/*
-+ * Primary cxgb3 API operations.
-+ * =============================
-+ */
-+
-+static int s3_push_frames(struct s3_conn *, int);
-+static int s3_send_reset(struct s3_conn *, int, struct sk_buff *);
-+static void t3_release_offload_resources(struct s3_conn *);
-+static void mk_close_req(struct s3_conn *);
-+
-+struct s3_conn *cxgb3i_c3cn_create(void)
-+{
-+ struct s3_conn *c3cn;
-+
-+ c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL);
-+ if (c3cn == NULL)
-+ return NULL;
-+
-+ c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn);
-+
-+ c3cn->flags = 0;
-+ spin_lock_init(&c3cn->lock);
-+ atomic_set(&c3cn->refcnt, 1);
-+ skb_queue_head_init(&c3cn->receive_queue);
-+ skb_queue_head_init(&c3cn->write_queue);
-+ setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn);
-+ rwlock_init(&c3cn->callback_lock);
-+
-+ return c3cn;
-+}
-+
-+static inline void s3_purge_write_queue(struct s3_conn *c3cn)
-+{
-+ struct sk_buff *skb;
-+
-+ while ((skb = __skb_dequeue(&c3cn->write_queue)))
-+ __kfree_skb(skb);
-+}
-+
-+static void c3cn_done(struct s3_conn *c3cn)
-+{
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ c3cn_put_port(c3cn);
-+ t3_release_offload_resources(c3cn);
-+ c3cn_set_state(c3cn, C3CN_STATE_CLOSE);
-+ c3cn->shutdown = C3CN_SHUTDOWN_MASK;
-+ cxgb3i_conn_closing(c3cn);
-+}
-+
-+static void c3cn_close(struct s3_conn *c3cn)
-+{
-+ int data_lost, old_state;
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ dst_confirm(c3cn->dst_cache);
-+
-+ spin_lock_bh(&c3cn->lock);
-+ c3cn->shutdown |= C3CN_SHUTDOWN_MASK;
-+
-+ /*
-+ * We need to flush the receive buffs. We do this only on the
-+ * descriptor close, not protocol-sourced closes, because the
-+ * reader process may not have drained the data yet! Make a note
-+ * of whether any received data will be lost so we can decide whether
-+ * to FIN or RST.
-+ */
-+ data_lost = skb_queue_len(&c3cn->receive_queue);
-+ __skb_queue_purge(&c3cn->receive_queue);
-+
-+ if (c3cn->state == C3CN_STATE_CLOSE)
-+ /* Nothing if we are already closed */
-+ c3cn_conn_debug("c3cn 0x%p, 0x%x, already closed.\n",
-+ c3cn, c3cn->state);
-+ else if (data_lost || c3cn->state == C3CN_STATE_SYN_SENT) {
-+ c3cn_conn_debug("c3cn 0x%p, 0x%x -> closing, send reset.\n",
-+ c3cn, c3cn->state);
-+ /* Unread data was tossed, zap the connection. */
-+ s3_send_reset(c3cn, CPL_ABORT_SEND_RST, NULL);
-+ goto unlock;
-+ } else if (c3cn->state == C3CN_STATE_ESTABLISHED) {
-+ c3cn_conn_debug("c3cn 0x%p, est. -> closing, send close_req.\n",
-+ c3cn);
-+ c3cn_set_state(c3cn, C3CN_STATE_CLOSING);
-+ mk_close_req(c3cn);
-+ }
-+
-+unlock:
-+ old_state = c3cn->state;
-+ c3cn_hold(c3cn); /* must last past the potential destroy() */
-+
-+ spin_unlock_bh(&c3cn->lock);
-+
-+ /*
-+ * There are no more user references at this point. Grab the
-+ * connection lock and finish the close.
-+ */
-+ local_bh_disable();
-+ spin_lock(&c3cn->lock);
-+
-+ /*
-+ * Because the connection was orphaned before the spin_lock()
-+ * either the backlog or a BH may have already destroyed it.
-+ * Bail out if so.
-+ */
-+ if (old_state != C3CN_STATE_CLOSE && c3cn->state == C3CN_STATE_CLOSE)
-+ goto out;
-+
-+ if (c3cn->state == C3CN_STATE_CLOSE)
-+ s3_purge_write_queue(c3cn);
-+
-+out:
-+ spin_unlock(&c3cn->lock);
-+ local_bh_enable();
-+ c3cn_put(c3cn);
-+}
-+
-+void cxgb3i_c3cn_release(struct s3_conn *c3cn)
-+{
-+ c3cn_conn_debug("c3cn 0x%p, s 0x%x, f 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+ if (likely(c3cn->state != C3CN_STATE_SYN_SENT))
-+ c3cn_close(c3cn);
-+ else
-+ c3cn_set_flag(c3cn, C3CN_CLOSE_NEEDED);
-+ c3cn_put(c3cn);
-+}
-+
-+
-+/*
-+ * Local utility routines used to implement primary cxgb3 API operations.
-+ * ======================================================================
-+ */
-+
-+static u32 s3_send_rx_credits(struct s3_conn *, u32, u32, int);
-+static int act_open(struct s3_conn *, struct net_device *);
-+static void mk_act_open_req(struct s3_conn *, struct sk_buff *,
-+ unsigned int, const struct l2t_entry *);
-+static void skb_entail(struct s3_conn *, struct sk_buff *, int);
-+
-+static inline void reset_wr_list(struct s3_conn *c3cn)
-+{
-+ c3cn->wr_pending_head = NULL;
-+}
-+
-+/*
-+ * Add a WR to a connections's list of pending WRs. This is a singly-linked
-+ * list of sk_buffs operating as a FIFO. The head is kept in wr_pending_head
-+ * and the tail in wr_pending_tail.
-+ */
-+static inline void enqueue_wr(struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ skb->sp = NULL;
-+
-+ /*
-+ * We want to take an extra reference since both us and the driver
-+ * need to free the packet before it's really freed. We know there's
-+ * just one user currently so we use atomic_set rather than skb_get
-+ * to avoid the atomic op.
-+ */
-+ atomic_set(&skb->users, 2);
-+
-+ if (!c3cn->wr_pending_head)
-+ c3cn->wr_pending_head = skb;
-+ else
-+ c3cn->wr_pending_tail->sp = (void *)skb;
-+ c3cn->wr_pending_tail = skb;
-+}
-+
-+/*
-+ * The next two functions calculate the option 0 value for a connection.
-+ */
-+static inline int compute_wscale(int win)
-+{
-+ int wscale = 0;
-+ while (wscale < 14 && (65535<<wscale) < win)
-+ wscale++;
-+ return wscale;
-+}
-+
-+static inline unsigned int calc_opt0h(struct s3_conn *c3cn)
-+{
-+ int wscale = compute_wscale(cxgb3_rcv_win);
-+ return V_KEEP_ALIVE(1) |
-+ F_TCAM_BYPASS |
-+ V_WND_SCALE(wscale) |
-+ V_MSS_IDX(c3cn->mss_idx);
-+}
-+
-+static inline unsigned int calc_opt0l(struct s3_conn *c3cn)
-+{
-+ return V_ULP_MODE(ULP_MODE_ISCSI) |
-+ V_RCV_BUFSIZ(cxgb3_rcv_win>>10);
-+}
-+
-+static inline void make_tx_data_wr(struct s3_conn *c3cn,
-+ struct sk_buff *skb, int len)
-+{
-+ struct tx_data_wr *req;
-+
-+ skb_reset_transport_header(skb);
-+ req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
-+ req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
-+ req->wr_lo = htonl(V_WR_TID(c3cn->tid));
-+ req->sndseq = htonl(c3cn->snd_nxt);
-+ /* len includes the length of any HW ULP additions */
-+ req->len = htonl(len);
-+ req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx));
-+ /* V_TX_ULP_SUBMODE sets both the mode and submode */
-+ req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) |
-+ V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
-+
-+ if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
-+
-+ req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
-+ V_TX_CPU_IDX(c3cn->qset));
-+
-+ /* Sendbuffer is in units of 32KB.
-+ */
-+ req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
-+ c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
-+ }
-+}
-+
-+/**
-+ * cxgb3_egress_dev - return the cxgb3 egress device
-+ * @root_dev: the root device anchoring the search
-+ * @c3cn: the connection used to determine egress port in bonding mode
-+ * @context: in bonding mode, indicates a connection set up or failover
-+ *
-+ * Return egress device or NULL if the egress device isn't one of our ports.
-+ *
-+ * Given a root network device it returns the physical egress device that is a
-+ * descendant of the root device. The root device may be either a physical
-+ * device, in which case it is the device returned, or a virtual device, such
-+ * as a VLAN or bonding device. In case of a bonding device the search
-+ * considers the decisions of the bonding device given its mode to locate the
-+ * correct egress device.
-+ */
-+static struct net_device *cxgb3_egress_dev(struct net_device *root_dev,
-+ struct s3_conn *c3cn,
-+ int context)
-+{
-+ while (root_dev) {
-+ if (root_dev->priv_flags & IFF_802_1Q_VLAN)
-+ root_dev = vlan_dev_real_dev(root_dev);
-+ else if (is_cxgb3_dev(root_dev))
-+ return root_dev;
-+ else
-+ return NULL;
-+ }
-+ return NULL;
-+}
-+
-+static struct rtable *find_route(__be32 saddr, __be32 daddr,
-+ __be16 sport, __be16 dport)
-+{
-+ struct rtable *rt;
-+ struct flowi fl = {
-+ .oif = 0,
-+ .nl_u = {
-+ .ip4_u = {
-+ .daddr = daddr,
-+ .saddr = saddr,
-+ .tos = 0 } },
-+ .proto = IPPROTO_TCP,
-+ .uli_u = {
-+ .ports = {
-+ .sport = sport,
-+ .dport = dport } } };
-+
-+ if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
-+ return NULL;
-+ return rt;
-+}
-+
-+int cxgb3i_c3cn_connect(struct s3_conn *c3cn, struct sockaddr_in *usin)
-+{
-+ struct rtable *rt;
-+ struct net_device *dev;
-+ struct cxgb3i_sdev_data *cdata;
-+ struct t3cdev *cdev;
-+ __be32 sipv4;
-+ int err;
-+
-+ if (usin->sin_family != AF_INET)
-+ return -EAFNOSUPPORT;
-+
-+ /* get a source port if one hasn't been provided */
-+ err = c3cn_get_port(c3cn);
-+ if (err)
-+ return err;
-+
-+ c3cn_conn_debug("c3cn 0x%p get port %u.\n",
-+ c3cn, ntohs(c3cn->saddr.sin_port));
-+
-+ c3cn->daddr.sin_port = usin->sin_port;
-+ c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr;
-+
-+ rt = find_route(c3cn->saddr.sin_addr.s_addr,
-+ c3cn->daddr.sin_addr.s_addr,
-+ c3cn->saddr.sin_port,
-+ c3cn->daddr.sin_port);
-+ if (rt == NULL) {
-+ c3cn_conn_debug("NO route to 0x%x, port %u.\n",
-+ c3cn->daddr.sin_addr.s_addr,
-+ ntohs(c3cn->daddr.sin_port));
-+ return -ENETUNREACH;
-+ }
-+
-+ if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
-+ c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n",
-+ c3cn->daddr.sin_addr.s_addr,
-+ ntohs(c3cn->daddr.sin_port));
-+ ip_rt_put(rt);
-+ return -ENETUNREACH;
-+ }
-+
-+ if (!c3cn->saddr.sin_addr.s_addr)
-+ c3cn->saddr.sin_addr.s_addr = rt->rt_src;
-+
-+ /* now commit destination to connection */
-+ c3cn->dst_cache = &rt->u.dst;
-+
-+ /* try to establish an offloaded connection */
-+ dev = cxgb3_egress_dev(c3cn->dst_cache->dev, c3cn, 0);
-+ if (dev == NULL) {
-+ c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn);
-+ return -ENETUNREACH;
-+ }
-+ cdata = NDEV2CDATA(dev);
-+ cdev = cdata->cdev;
-+
-+ sipv4 = cxgb3i_get_private_ipv4addr(dev);
-+ if (!sipv4) {
-+ c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn);
-+ sipv4 = c3cn->saddr.sin_addr.s_addr;
-+ cxgb3i_set_private_ipv4addr(dev, sipv4);
-+ } else
-+ c3cn->saddr.sin_addr.s_addr = sipv4;
-+
-+ c3cn_conn_debug("c3cn 0x%p, %u.%u.%u.%u,%u-%u.%u.%u.%u,%u SYN_SENT.\n",
-+ c3cn, NIPQUAD(c3cn->saddr.sin_addr.s_addr),
-+ ntohs(c3cn->saddr.sin_port),
-+ NIPQUAD(c3cn->daddr.sin_addr.s_addr),
-+ ntohs(c3cn->daddr.sin_port));
-+
-+ c3cn_set_state(c3cn, C3CN_STATE_SYN_SENT);
-+
-+ if (!act_open(c3cn, dev))
-+ return 0;
-+
-+ /*
-+ * If we get here, we don't have an offload connection so simply
-+ * return a failure.
-+ */
-+ err = -ENOTSUPP;
-+
-+ /*
-+ * This trashes the connection and releases the local port,
-+ * if necessary.
-+ */
-+ c3cn_conn_debug("c3cn 0x%p -> CLOSE.\n", c3cn);
-+ c3cn_set_state(c3cn, C3CN_STATE_CLOSE);
-+ ip_rt_put(rt);
-+ c3cn_put_port(c3cn);
-+ c3cn->daddr.sin_port = 0;
-+ return err;
-+}
-+
-+/*
-+ * Set of states for which we should return RX credits.
-+ */
-+#define CREDIT_RETURN_STATE (C3CN_STATE_ESTABLISHED)
-+
-+/*
-+ * Called after some received data has been read. It returns RX credits
-+ * to the HW for the amount of data processed.
-+ */
-+void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied)
-+{
-+ struct t3cdev *cdev;
-+ int must_send;
-+ u32 credits, dack = 0;
-+
-+ if (!c3cn_in_state(c3cn, CREDIT_RETURN_STATE))
-+ return;
-+
-+ credits = c3cn->copied_seq - c3cn->rcv_wup;
-+ if (unlikely(!credits))
-+ return;
-+
-+ cdev = c3cn->cdev;
-+
-+ if (unlikely(cxgb3_rx_credit_thres == 0))
-+ return;
-+
-+ dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
-+
-+ /*
-+ * For coalescing to work effectively ensure the receive window has
-+ * at least 16KB left.
-+ */
-+ must_send = credits + 16384 >= cxgb3_rcv_win;
-+
-+ if (must_send || credits >= cxgb3_rx_credit_thres)
-+ c3cn->rcv_wup += s3_send_rx_credits(c3cn, credits, dack,
-+ must_send);
-+}
-+
-+/*
-+ * Generic ARP failure handler that discards the buffer.
-+ */
-+static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb)
-+{
-+ kfree_skb(skb);
-+}
-+
-+/*
-+ * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
-+ * connection's send queue and sends them on to T3. Must be called with the
-+ * connection's lock held. Returns the amount of send buffer space that was
-+ * freed as a result of sending queued data to T3.
-+ */
-+static int s3_push_frames(struct s3_conn *c3cn, int req_completion)
-+{
-+ int total_size = 0;
-+ struct sk_buff *skb;
-+ struct t3cdev *cdev;
-+ struct cxgb3i_sdev_data *cdata;
-+
-+ if (unlikely(c3cn_in_state(c3cn,
-+ C3CN_STATE_SYN_SENT | C3CN_STATE_CLOSE)))
-+ return 0;
-+
-+ /*
-+ * We shouldn't really be called at all after an abort but check just
-+ * in case.
-+ */
-+ if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN)))
-+ return 0;
-+
-+ cdev = c3cn->cdev;
-+ cdata = CXGB3_SDEV_DATA(cdev);
-+
-+ while (c3cn->wr_avail
-+ && (skb = skb_peek(&c3cn->write_queue)) != NULL
-+ && !c3cn_flag(c3cn, C3CN_TX_WAIT_IDLE)) {
-+
-+ int len = skb->len; /* length before skb_push */
-+ int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
-+ int wrs_needed = skb_wrs[frags];
-+
-+ if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
-+ wrs_needed = 1;
-+
-+ WARN_ON(frags >= ARRAY_SIZE(skb_wrs) || wrs_needed < 1);
-+
-+ if (c3cn->wr_avail < wrs_needed)
-+ break;
-+
-+ __skb_unlink(skb, &c3cn->write_queue);
-+ skb->priority = CPL_PRIORITY_DATA;
-+ skb->csum = wrs_needed; /* remember this until the WR_ACK */
-+ c3cn->wr_avail -= wrs_needed;
-+ c3cn->wr_unacked += wrs_needed;
-+ enqueue_wr(c3cn, skb);
-+
-+ if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
-+ len += ulp_extra_len(skb);
-+ make_tx_data_wr(c3cn, skb, len);
-+ c3cn->snd_nxt += len;
-+ if ((req_completion
-+ && c3cn->wr_unacked == wrs_needed)
-+ || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL)
-+ || c3cn->wr_unacked >= c3cn->wr_max / 2) {
-+ struct work_request_hdr *wr = cplhdr(skb);
-+
-+ wr->wr_hi |= htonl(F_WR_COMPL);
-+ c3cn->wr_unacked = 0;
-+ }
-+ CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR;
-+ } else if (skb->data[0] == FW_WROPCODE_OFLD_CLOSE_CON)
-+ c3cn_set_flag(c3cn, C3CN_CLOSE_CON_REQUESTED);
-+
-+ total_size += skb->truesize;
-+ set_arp_failure_handler(skb, arp_failure_discard);
-+ l2t_send(cdev, skb, c3cn->l2t);
-+ }
-+ return total_size;
-+}
-+
-+/*
-+ * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant
-+ * and send it along.
-+ */
-+static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb)
-+{
-+ struct cpl_abort_req *req = cplhdr(skb);
-+
-+ c3cn_conn_debug("tdev 0x%p.\n", cdev);
-+
-+ req->cmd = CPL_ABORT_NO_RST;
-+ cxgb3_ofld_send(cdev, skb);
-+}
-+
-+/*
-+ * Send an ABORT_REQ message. Cannot fail. This routine makes sure we do
-+ * not send multiple ABORT_REQs for the same connection and also that we do
-+ * not try to send a message after the connection has closed. Returns 1 if
-+ * an ABORT_REQ wasn't generated after all, 0 otherwise.
-+ */
-+static int s3_send_reset(struct s3_conn *c3cn, int mode,
-+ struct sk_buff *skb)
-+{
-+ struct cpl_abort_req *req;
-+ unsigned int tid = c3cn->tid;
-+
-+ if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN) || !c3cn->cdev)) {
-+ if (skb)
-+ __kfree_skb(skb);
-+ return 1;
-+ }
-+
-+ c3cn_conn_debug("c3cn 0x%p, mode %d, flag ABORT_RPL + ABORT_SHUT.\n",
-+ c3cn, mode);
-+
-+ c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING);
-+ c3cn_set_flag(c3cn, C3CN_ABORT_SHUTDOWN);
-+
-+ /* Purge the send queue so we don't send anything after an abort. */
-+ s3_purge_write_queue(c3cn);
-+
-+ if (!skb)
-+ skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
-+ skb->priority = CPL_PRIORITY_DATA;
-+ set_arp_failure_handler(skb, abort_arp_failure);
-+
-+ req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
-+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
-+ req->wr.wr_lo = htonl(V_WR_TID(tid));
-+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
-+ req->rsvd0 = htonl(c3cn->snd_nxt);
-+ req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT);
-+ req->cmd = mode;
-+
-+ l2t_send(c3cn->cdev, skb, c3cn->l2t);
-+ return 0;
-+}
-+
-+/*
-+ * Add a list of skbs to a connection send queue. This interface is intended
-+ * for use by in-kernel ULPs. The skbs must comply with the max size limit of
-+ * the device and have a headroom of at least TX_HEADER_LEN bytes.
-+ */
-+int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb, int flags)
-+{
-+ struct sk_buff *next;
-+ int err, copied = 0;
-+
-+ spin_lock_bh(&c3cn->lock);
-+
-+ if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED)) {
-+ err = -EAGAIN;
-+ goto out_err;
-+ }
-+
-+ err = -EPIPE;
-+ if (c3cn->err || (c3cn->shutdown & C3CN_SEND_SHUTDOWN))
-+ goto out_err;
-+
-+ while (skb) {
-+ if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
-+ c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
-+ err = -EINVAL;
-+ goto out_err;
-+ }
-+
-+ next = skb->next;
-+ skb->next = NULL;
-+ skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
-+ copied += skb->len;
-+ c3cn->write_seq += skb->len + ulp_extra_len(skb);
-+ skb = next;
-+ }
-+done:
-+ if (likely(skb_queue_len(&c3cn->write_queue)))
-+ s3_push_frames(c3cn, 1);
-+ spin_unlock_bh(&c3cn->lock);
-+ return copied;
-+
-+out_err:
-+ if (copied == 0 && err == -EPIPE)
-+ copied = c3cn->err ? c3cn->err : -EPIPE;
-+ goto done;
-+}
-+
-+/*
-+ * Low-level utility routines for primary API functions.
-+ * =====================================================
-+ */
-+/* routines to implement CPL message processing */
-+static void c3cn_act_establish(struct s3_conn *, struct sk_buff *);
-+static void active_open_failed(struct s3_conn *, struct sk_buff *);
-+static void wr_ack(struct s3_conn *, struct sk_buff *);
-+static void do_peer_fin(struct s3_conn *, struct sk_buff *);
-+static void process_abort_req(struct s3_conn *, struct sk_buff *);
-+static void process_abort_rpl(struct s3_conn *, struct sk_buff *);
-+static void process_close_con_rpl(struct s3_conn *, struct sk_buff *);
-+static void process_rx_iscsi_hdr(struct s3_conn *, struct sk_buff *);
-+
-+static struct sk_buff *__get_cpl_reply_skb(struct sk_buff *, size_t, gfp_t);
-+
-+static void fail_act_open(struct s3_conn *, int);
-+static void init_offload_conn(struct s3_conn *, struct t3cdev *,
-+ struct dst_entry *);
-+
-+/*
-+ * Insert a connection into the TID table and take an extra reference.
-+ */
-+static inline void c3cn_insert_tid(struct cxgb3i_sdev_data *cdata,
-+ struct s3_conn *c3cn,
-+ unsigned int tid)
-+{
-+ c3cn_hold(c3cn);
-+ cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid);
-+}
-+
-+static inline void free_atid(struct t3cdev *cdev, unsigned int tid)
-+{
-+ struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid);
-+ if (c3cn)
-+ c3cn_put(c3cn);
-+}
-+
-+/*
-+ * This function is intended for allocations of small control messages.
-+ * Such messages go as immediate data and usually the pakets are freed
-+ * immediately. We maintain a cache of one small sk_buff and use it whenever
-+ * it is available (has a user count of 1). Otherwise we get a fresh buffer.
-+ */
-+#define CTRL_SKB_LEN 120
-+
-+static struct sk_buff *alloc_ctrl_skb(const struct s3_conn *c3cn,
-+ int len)
-+{
-+ struct sk_buff *skb = c3cn->ctrl_skb_cache;
-+
-+ if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
-+ __skb_trim(skb, 0);
-+ atomic_set(&skb->users, 2);
-+ } else if (likely(!in_atomic()))
-+ skb = alloc_skb(len, GFP_ATOMIC | __GFP_NOFAIL);
-+ else
-+ skb = alloc_skb(len, GFP_ATOMIC);
-+ return skb;
-+}
-+
-+/*
-+ * Handle an ARP failure for an active open.
-+ */
-+static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
-+{
-+ struct s3_conn *c3cn = (struct s3_conn *)skb->sk;
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
-+
-+ c3cn_hold(c3cn);
-+ spin_lock(&c3cn->lock);
-+ if (c3cn->state == C3CN_STATE_SYN_SENT) {
-+ fail_act_open(c3cn, EHOSTUNREACH);
-+ __kfree_skb(skb);
-+ }
-+ spin_unlock(&c3cn->lock);
-+ c3cn_put(c3cn);
-+}
-+
-+/*
-+ * Send an active open request.
-+ */
-+static int act_open(struct s3_conn *c3cn, struct net_device *dev)
-+{
-+ struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev);
-+ struct t3cdev *cdev = cdata->cdev;
-+ struct dst_entry *dst = c3cn->dst_cache;
-+ struct sk_buff *skb;
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+ /*
-+ * Initialize connection data. Note that the flags and ULP mode are
-+ * initialized higher up ...
-+ */
-+ c3cn->dev = dev;
-+ c3cn->cdev = cdev;
-+ c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn);
-+ if (c3cn->tid < 0)
-+ goto out_err;
-+
-+ c3cn->qset = 0;
-+ c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev);
-+ if (!c3cn->l2t)
-+ goto free_tid;
-+
-+ skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL);
-+ if (!skb)
-+ goto free_l2t;
-+
-+ skb->sk = (struct sock *)c3cn;
-+ set_arp_failure_handler(skb, act_open_req_arp_failure);
-+
-+ c3cn_hold(c3cn);
-+
-+ init_offload_conn(c3cn, cdev, dst);
-+ c3cn->err = 0;
-+ c3cn_reset_flag(c3cn, C3CN_DONE);
-+
-+ mk_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
-+ l2t_send(cdev, skb, c3cn->l2t);
-+ return 0;
-+
-+free_l2t:
-+ l2t_release(L2DATA(cdev), c3cn->l2t);
-+free_tid:
-+ free_atid(cdev, c3cn->tid);
-+ c3cn->tid = 0;
-+out_err:
-+ return -1;
-+}
-+
-+/*
-+ * Close a connection by sending a CPL_CLOSE_CON_REQ message. Cannot fail
-+ * under any circumstances. We take the easy way out and always queue the
-+ * message to the write_queue. We can optimize the case where the queue is
-+ * already empty though the optimization is probably not worth it.
-+ */
-+static void mk_close_req(struct s3_conn *c3cn)
-+{
-+ struct sk_buff *skb;
-+ struct cpl_close_con_req *req;
-+ unsigned int tid = c3cn->tid;
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ skb = alloc_skb(sizeof(struct cpl_close_con_req),
-+ GFP_KERNEL | __GFP_NOFAIL);
-+ req = (struct cpl_close_con_req *)__skb_put(skb, sizeof(*req));
-+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
-+ req->wr.wr_lo = htonl(V_WR_TID(tid));
-+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
-+ req->rsvd = htonl(c3cn->write_seq);
-+
-+ skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND);
-+ if (c3cn->state != C3CN_STATE_SYN_SENT)
-+ s3_push_frames(c3cn, 1);
-+}
-+
-+static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
-+ int flags)
-+{
-+ CXGB3_SKB_CB(skb)->seq = c3cn->write_seq;
-+ CXGB3_SKB_CB(skb)->flags = flags;
-+ __skb_queue_tail(&c3cn->write_queue, skb);
-+}
-+
-+/*
-+ * Send RX credits through an RX_DATA_ACK CPL message. If nofail is 0 we are
-+ * permitted to return without sending the message in case we cannot allocate
-+ * an sk_buff. Returns the number of credits sent.
-+ */
-+static u32 s3_send_rx_credits(struct s3_conn *c3cn, u32 credits, u32 dack,
-+ int nofail)
-+{
-+ struct sk_buff *skb;
-+ struct cpl_rx_data_ack *req;
-+
-+ skb = (nofail ? alloc_ctrl_skb(c3cn, sizeof(*req))
-+ : alloc_skb(sizeof(*req), GFP_ATOMIC));
-+ if (!skb)
-+ return 0;
-+
-+ req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
-+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
-+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, c3cn->tid));
-+ req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
-+ skb->priority = CPL_PRIORITY_ACK;
-+ cxgb3_ofld_send(c3cn->cdev, skb);
-+ return credits;
-+}
-+
-+static void mk_act_open_req(struct s3_conn *c3cn, struct sk_buff *skb,
-+ unsigned int atid, const struct l2t_entry *e)
-+{
-+ struct cpl_act_open_req *req;
-+
-+ c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn, atid);
-+
-+ skb->priority = CPL_PRIORITY_SETUP;
-+ req = (struct cpl_act_open_req *)__skb_put(skb, sizeof(*req));
-+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
-+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
-+ req->local_port = c3cn->saddr.sin_port;
-+ req->peer_port = c3cn->daddr.sin_port;
-+ req->local_ip = c3cn->saddr.sin_addr.s_addr;
-+ req->peer_ip = c3cn->daddr.sin_addr.s_addr;
-+ req->opt0h = htonl(calc_opt0h(c3cn) | V_L2T_IDX(e->idx) |
-+ V_TX_CHANNEL(e->smt_idx));
-+ req->opt0l = htonl(calc_opt0l(c3cn));
-+ req->params = 0;
-+}
-+
-+/*
-+ * Definitions and declarations for CPL handler functions.
-+ * =======================================================
-+ */
-+
-+/*
-+ * Similar to process_cpl_msg() but takes an extra connection reference around
-+ * the call to the handler. Should be used if the handler may drop a
-+ * connection reference.
-+ */
-+static inline void process_cpl_msg_ref(void (*fn) (struct s3_conn *,
-+ struct sk_buff *),
-+ struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ c3cn_hold(c3cn);
-+ process_cpl_msg(fn, c3cn, skb);
-+ c3cn_put(c3cn);
-+}
-+
-+/*
-+ * Return whether a failed active open has allocated a TID
-+ */
-+static inline int act_open_has_tid(int status)
-+{
-+ return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST &&
-+ status != CPL_ERR_ARP_MISS;
-+}
-+
-+/*
-+ * Returns true if a connection cannot accept new Rx data.
-+ */
-+static inline int c3cn_no_receive(const struct s3_conn *c3cn)
-+{
-+ return c3cn->shutdown & C3CN_RCV_SHUTDOWN;
-+}
-+
-+/*
-+ * A helper function that aborts a connection and increments the given MIB
-+ * counter. The supplied skb is used to generate the ABORT_REQ message if
-+ * possible. Must be called with softirqs disabled.
-+ */
-+static inline void abort_conn(struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ struct sk_buff *abort_skb;
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ abort_skb = __get_cpl_reply_skb(skb, sizeof(struct cpl_abort_req),
-+ GFP_ATOMIC);
-+ if (abort_skb)
-+ s3_send_reset(c3cn, CPL_ABORT_SEND_RST, abort_skb);
-+}
-+
-+/*
-+ * Returns whether an ABORT_REQ_RSS message is a negative advice.
-+ */
-+static inline int is_neg_adv_abort(unsigned int status)
-+{
-+ return status == CPL_ERR_RTX_NEG_ADVICE ||
-+ status == CPL_ERR_PERSIST_NEG_ADVICE;
-+}
-+
-+/*
-+ * CPL handler functions.
-+ * ======================
-+ */
-+
-+/*
-+ * Process a CPL_ACT_ESTABLISH message.
-+ */
-+static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
-+ void *ctx)
-+{
-+ struct cpl_act_establish *req = cplhdr(skb);
-+ unsigned int tid = GET_TID(req);
-+ unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
-+ struct s3_conn *c3cn = ctx;
-+ struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
-+
-+ c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
-+ tid, c3cn, c3cn->state, c3cn->flags);
-+ /*
-+ * It's OK if the TID is currently in use, the owning connection may
-+ * have backlogged its last CPL message(s). Just take it away.
-+ */
-+ c3cn->tid = tid;
-+ c3cn_insert_tid(cdata, c3cn, tid);
-+ free_atid(cdev, atid);
-+
-+ c3cn->qset = G_QNUM(ntohl(skb->csum));
-+
-+ process_cpl_msg(c3cn_act_establish, c3cn, skb);
-+ return 0;
-+}
-+
-+/*
-+ * Process an ACT_OPEN_RPL CPL message.
-+ */
-+static int do_act_open_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
-+{
-+ struct s3_conn *c3cn = ctx;
-+ struct cpl_act_open_rpl *rpl = cplhdr(skb);
-+
-+ c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
-+ rpl->status, c3cn, c3cn->state, c3cn->flags);
-+
-+ if (act_open_has_tid(rpl->status))
-+ cxgb3_queue_tid_release(cdev, GET_TID(rpl));
-+
-+ process_cpl_msg_ref(active_open_failed, c3cn, skb);
-+ return 0;
-+}
-+
-+/*
-+ * Handler RX_ISCSI_HDR CPL messages.
-+ */
-+static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
-+{
-+ struct s3_conn *c3cn = ctx;
-+ process_cpl_msg(process_rx_iscsi_hdr, c3cn, skb);
-+ return 0;
-+}
-+
-+/*
-+ * Handler for TX_DATA_ACK CPL messages.
-+ */
-+static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
-+{
-+ struct s3_conn *c3cn = ctx;
-+
-+ process_cpl_msg(wr_ack, c3cn, skb);
-+ return 0;
-+}
-+
-+/*
-+ * Handler for PEER_CLOSE CPL messages.
-+ */
-+static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
-+{
-+ struct s3_conn *c3cn = ctx;
-+
-+ c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+ process_cpl_msg_ref(do_peer_fin, c3cn, skb);
-+ return 0;
-+}
-+
-+/*
-+ * Handle an ABORT_REQ_RSS CPL message.
-+ */
-+static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
-+{
-+ const struct cpl_abort_req_rss *req = cplhdr(skb);
-+ struct s3_conn *c3cn = ctx;
-+
-+ c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ if (is_neg_adv_abort(req->status)) {
-+ __kfree_skb(skb);
-+ return 0;
-+ }
-+
-+ process_cpl_msg_ref(process_abort_req, c3cn, skb);
-+ return 0;
-+}
-+
-+/*
-+ * Handle an ABORT_RPL_RSS CPL message.
-+ */
-+static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
-+{
-+ struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
-+ struct s3_conn *c3cn = ctx;
-+
-+ c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
-+ rpl->status, c3cn, c3cn ? c3cn->state : 0,
-+ c3cn ? c3cn->flags : 0UL);
-+
-+ /*
-+ * Ignore replies to post-close aborts indicating that the abort was
-+ * requested too late. These connections are terminated when we get
-+ * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
-+ * arrives the TID is either no longer used or it has been recycled.
-+ */
-+ if (rpl->status == CPL_ERR_ABORT_FAILED)
-+ goto discard;
-+
-+ /*
-+ * Sometimes we've already closed the connection, e.g., a post-close
-+ * abort races with ABORT_REQ_RSS, the latter frees the connection
-+ * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
-+ * but FW turns the ABORT_REQ into a regular one and so we get
-+ * ABORT_RPL_RSS with status 0 and no connection. Only on T3A.
-+ */
-+ if (!c3cn)
-+ goto discard;
-+
-+ process_cpl_msg_ref(process_abort_rpl, c3cn, skb);
-+ return 0;
-+
-+discard:
-+ __kfree_skb(skb);
-+ return 0;
-+}
-+
-+/*
-+ * Handler for CLOSE_CON_RPL CPL messages.
-+ */
-+static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
-+ void *ctx)
-+{
-+ struct s3_conn *c3cn = ctx;
-+
-+ c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ process_cpl_msg_ref(process_close_con_rpl, c3cn, skb);
-+ return 0;
-+}
-+
-+/*
-+ * Definitions and declarations for CPL message processing.
-+ * ========================================================
-+ */
-+
-+static void make_established(struct s3_conn *, u32, unsigned int);
-+static void act_open_retry_timer(unsigned long);
-+static void mk_act_open_req(struct s3_conn *, struct sk_buff *,
-+ unsigned int, const struct l2t_entry *);
-+static int act_open_rpl_status_to_errno(int);
-+static void handle_excess_rx(struct s3_conn *, struct sk_buff *);
-+static int abort_status_to_errno(struct s3_conn *, int, int *);
-+static void send_abort_rpl(struct sk_buff *, struct t3cdev *, int);
-+static struct sk_buff *get_cpl_reply_skb(struct sk_buff *, size_t, gfp_t);
-+
-+/*
-+ * Dequeue and return the first unacknowledged's WR on a connections's pending
-+ * list.
-+ */
-+static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
-+{
-+ struct sk_buff *skb = c3cn->wr_pending_head;
-+
-+ if (likely(skb)) {
-+ /* Don't bother clearing the tail */
-+ c3cn->wr_pending_head = (struct sk_buff *)skb->sp;
-+ skb->sp = NULL;
-+ }
-+ return skb;
-+}
-+
-+/*
-+ * Return the first pending WR without removing it from the list.
-+ */
-+static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
-+{
-+ return c3cn->wr_pending_head;
-+}
-+
-+static inline void free_wr_skb(struct sk_buff *skb)
-+{
-+ kfree_skb(skb);
-+}
-+
-+static void purge_wr_queue(struct s3_conn *c3cn)
-+{
-+ struct sk_buff *skb;
-+ while ((skb = dequeue_wr(c3cn)) != NULL)
-+ free_wr_skb(skb);
-+}
-+
-+static inline void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
-+ int cmd)
-+{
-+ struct cpl_abort_rpl *rpl = cplhdr(skb);
-+
-+ rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
-+ rpl->wr.wr_lo = htonl(V_WR_TID(tid));
-+ OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
-+ rpl->cmd = cmd;
-+}
-+
-+/*
-+ * CPL message processing ...
-+ * ==========================
-+ */
-+
-+/*
-+ * Updates connection state from an active establish CPL message. Runs with
-+ * the connection lock held.
-+ */
-+static void c3cn_act_establish(struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ struct cpl_act_establish *req = cplhdr(skb);
-+ u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ if (unlikely(c3cn->state != C3CN_STATE_SYN_SENT))
-+ printk(KERN_ERR "TID %u expected SYN_SENT, found %d\n",
-+ c3cn->tid, c3cn->state);
-+
-+ c3cn->copied_seq = c3cn->rcv_wup = c3cn->rcv_nxt = rcv_isn;
-+ make_established(c3cn, ntohl(req->snd_isn), ntohs(req->tcp_opt));
-+
-+ if (unlikely(c3cn_flag(c3cn, C3CN_CLOSE_NEEDED))) {
-+ /* upper layer has requested closing */
-+ abort_conn(c3cn, skb);
-+ return;
-+ }
-+
-+ __kfree_skb(skb);
-+ if (s3_push_frames(c3cn, 1))
-+ cxgb3i_conn_tx_open(c3cn);
-+}
-+
-+/*
-+ * Handle active open failures.
-+ */
-+static void active_open_failed(struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ struct cpl_act_open_rpl *rpl = cplhdr(skb);
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ if (rpl->status == CPL_ERR_CONN_EXIST &&
-+ c3cn->retry_timer.function != act_open_retry_timer) {
-+ c3cn->retry_timer.function = act_open_retry_timer;
-+ c3cn_reset_timer(c3cn, &c3cn->retry_timer,
-+ jiffies + HZ / 2);
-+ } else
-+ fail_act_open(c3cn, act_open_rpl_status_to_errno(rpl->status));
-+ __kfree_skb(skb);
-+}
-+
-+/*
-+ * Process received pdu for a connection.
-+ */
-+static void process_rx_iscsi_hdr(struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
-+ struct cpl_iscsi_hdr_norss data_cpl;
-+ struct cpl_rx_data_ddp_norss ddp_cpl;
-+ unsigned int hdr_len, data_len, status;
-+ unsigned int len;
-+ int err;
-+
-+ if (unlikely(c3cn_no_receive(c3cn))) {
-+ handle_excess_rx(c3cn, skb);
-+ return;
-+ }
-+
-+ CXGB3_SKB_CB(skb)->seq = ntohl(hdr_cpl->seq);
-+ CXGB3_SKB_CB(skb)->flags = 0;
-+
-+ skb_reset_transport_header(skb);
-+ __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
-+
-+ len = hdr_len = ntohs(hdr_cpl->len);
-+ /* msg coalesce is off or not enough data received */
-+ if (skb->len <= hdr_len) {
-+ printk(KERN_ERR "%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
-+ c3cn->cdev->name, c3cn->tid, skb->len, hdr_len);
-+ goto abort_conn;
-+ }
-+
-+ err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
-+ sizeof(ddp_cpl));
-+ if (err < 0)
-+ goto abort_conn;
-+
-+ skb_ulp_mode(skb) = ULP2_FLAG_DATA_READY;
-+ skb_ulp_pdulen(skb) = ntohs(ddp_cpl.len);
-+ skb_ulp_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
-+ status = ntohl(ddp_cpl.ddp_status);
-+
-+ if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
-+ skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
-+ if (status & (1 << RX_DDP_STATUS_DCRC_SHIFT))
-+ skb_ulp_mode(skb) |= ULP2_FLAG_DCRC_ERROR;
-+ if (status & (1 << RX_DDP_STATUS_PAD_SHIFT))
-+ skb_ulp_mode(skb) |= ULP2_FLAG_PAD_ERROR;
-+
-+ if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
-+ err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
-+ if (err < 0)
-+ goto abort_conn;
-+ data_len = ntohs(data_cpl.len);
-+ len += sizeof(data_cpl) + data_len;
-+ } else if (status & (1 << RX_DDP_STATUS_DDP_SHIFT))
-+ skb_ulp_mode(skb) |= ULP2_FLAG_DATA_DDPED;
-+
-+ c3cn->rcv_nxt = ntohl(ddp_cpl.seq) + skb_ulp_pdulen(skb);
-+ __pskb_trim(skb, len);
-+ __skb_queue_tail(&c3cn->receive_queue, skb);
-+ cxgb3i_conn_pdu_ready(c3cn);
-+
-+ return;
-+
-+abort_conn:
-+ s3_send_reset(c3cn, CPL_ABORT_SEND_RST, NULL);
-+ __kfree_skb(skb);
-+}
-+
-+/*
-+ * Process an acknowledgment of WR completion. Advance snd_una and send the
-+ * next batch of work requests from the write queue.
-+ */
-+static void wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
-+{
-+ struct cpl_wr_ack *hdr = cplhdr(skb);
-+ unsigned int credits = ntohs(hdr->credits);
-+ u32 snd_una = ntohl(hdr->snd_una);
-+
-+ c3cn->wr_avail += credits;
-+ if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
-+ c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
-+
-+ while (credits) {
-+ struct sk_buff *p = peek_wr(c3cn);
-+
-+ if (unlikely(!p)) {
-+ printk(KERN_ERR "%u WR_ACK credits for TID %u with "
-+ "nothing pending, state %u\n",
-+ credits, c3cn->tid, c3cn->state);
-+ break;
-+ }
-+ if (unlikely(credits < p->csum)) {
-+ p->csum -= credits;
-+ break;
-+ } else {
-+ dequeue_wr(c3cn);
-+ credits -= p->csum;
-+ free_wr_skb(p);
-+ }
-+ }
-+
-+ if (unlikely(before(snd_una, c3cn->snd_una)))
-+ goto out_free;
-+
-+ if (c3cn->snd_una != snd_una) {
-+ c3cn->snd_una = snd_una;
-+ dst_confirm(c3cn->dst_cache);
-+ if (c3cn->snd_una == c3cn->snd_nxt)
-+ c3cn_reset_flag(c3cn, C3CN_TX_WAIT_IDLE);
-+ }
-+
-+ if (skb_queue_len(&c3cn->write_queue) && s3_push_frames(c3cn, 0))
-+ cxgb3i_conn_tx_open(c3cn);
-+out_free:
-+ __kfree_skb(skb);
-+}
-+
-+/*
-+ * Handle a peer FIN.
-+ */
-+static void do_peer_fin(struct s3_conn *c3cn, struct sk_buff *skb)
-+{
-+ int keep = 0;
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
-+ goto out;
-+
-+ c3cn->shutdown |= C3CN_RCV_SHUTDOWN;
-+ c3cn_set_flag(c3cn, C3CN_DONE);
-+
-+ switch (c3cn->state) {
-+ case C3CN_STATE_ESTABLISHED:
-+ break;
-+ case C3CN_STATE_CLOSING:
-+ c3cn_done(c3cn);
-+ break;
-+ default:
-+ printk(KERN_ERR
-+ "%s: TID %u received PEER_CLOSE in bad state %d\n",
-+ c3cn->cdev->name, c3cn->tid, c3cn->state);
-+ }
-+
-+ cxgb3i_conn_closing(c3cn);
-+out:
-+ if (!keep)
-+ __kfree_skb(skb);
-+}
-+
-+/*
-+ * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
-+ * request except that we need to reply to it.
-+ */
-+static void process_abort_req(struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ int rst_status = CPL_ABORT_NO_RST;
-+ const struct cpl_abort_req_rss *req = cplhdr(skb);
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ if (!c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD)) {
-+ c3cn_set_flag(c3cn, C3CN_ABORT_REQ_RCVD);
-+ c3cn_set_flag(c3cn, C3CN_ABORT_SHUTDOWN);
-+ __kfree_skb(skb);
-+ return;
-+ }
-+ c3cn_reset_flag(c3cn, C3CN_ABORT_REQ_RCVD);
-+
-+ /*
-+ * Three cases to consider:
-+ * a) We haven't sent an abort_req; close the connection.
-+ * b) We have sent a post-close abort_req that will get to TP too late
-+ * and will generate a CPL_ERR_ABORT_FAILED reply. The reply will
-+ * be ignored and the connection should be closed now.
-+ * c) We have sent a regular abort_req that will get to TP too late.
-+ * That will generate an abort_rpl with status 0, wait for it.
-+ */
-+ send_abort_rpl(skb, c3cn->cdev, rst_status);
-+
-+ if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
-+ c3cn->err =
-+ abort_status_to_errno(c3cn, req->status, &rst_status);
-+
-+ c3cn_done(c3cn);
-+ }
-+}
-+
-+/*
-+ * Process abort replies. We only process these messages if we anticipate
-+ * them as the coordination between SW and HW in this area is somewhat lacking
-+ * and sometimes we get ABORT_RPLs after we are done with the connection that
-+ * originated the ABORT_REQ.
-+ */
-+static void process_abort_rpl(struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
-+ if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_RCVD))
-+ c3cn_set_flag(c3cn, C3CN_ABORT_RPL_RCVD);
-+ else {
-+ c3cn_reset_flag(c3cn, C3CN_ABORT_RPL_RCVD);
-+ c3cn_reset_flag(c3cn, C3CN_ABORT_RPL_PENDING);
-+ BUG_ON(c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD));
-+ c3cn_done(c3cn);
-+ }
-+ }
-+ __kfree_skb(skb);
-+}
-+
-+/*
-+ * Process a peer ACK to our FIN.
-+ */
-+static void process_close_con_rpl(struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ struct cpl_close_con_rpl *rpl = cplhdr(skb);
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ c3cn->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
-+
-+ if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
-+ goto out;
-+
-+ if (c3cn->state == C3CN_STATE_CLOSING) {
-+ c3cn_done(c3cn);
-+ } else
-+ printk(KERN_ERR
-+ "%s: TID %u received CLOSE_CON_RPL in bad state %d\n",
-+ c3cn->cdev->name, c3cn->tid, c3cn->state);
-+out:
-+ kfree_skb(skb);
-+}
-+
-+/*
-+ * Random utility functions for CPL message processing ...
-+ * =======================================================
-+ */
-+
-+/**
-+ * find_best_mtu - find the entry in the MTU table closest to an MTU
-+ * @d: TOM state
-+ * @mtu: the target MTU
-+ *
-+ * Returns the index of the value in the MTU table that is closest to but
-+ * does not exceed the target MTU.
-+ */
-+static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
-+{
-+ int i = 0;
-+
-+ while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
-+ ++i;
-+ return i;
-+}
-+
-+static unsigned int select_mss(struct s3_conn *c3cn, unsigned int pmtu)
-+{
-+ unsigned int idx;
-+ struct dst_entry *dst = c3cn->dst_cache;
-+ struct t3cdev *cdev = c3cn->cdev;
-+ const struct t3c_data *td = T3C_DATA(cdev);
-+ u16 advmss = dst_metric(dst, RTAX_ADVMSS);
-+
-+ if (advmss > pmtu - 40)
-+ advmss = pmtu - 40;
-+ if (advmss < td->mtus[0] - 40)
-+ advmss = td->mtus[0] - 40;
-+ idx = find_best_mtu(td, advmss + 40);
-+ return idx;
-+}
-+
-+static void fail_act_open(struct s3_conn *c3cn, int errno)
-+{
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
-+ c3cn, c3cn->state, c3cn->flags);
-+
-+ c3cn->err = errno;
-+ c3cn_done(c3cn);
-+}
-+
-+/*
-+ * Assign offload parameters to some connection fields.
-+ */
-+static void init_offload_conn(struct s3_conn *c3cn,
-+ struct t3cdev *cdev,
-+ struct dst_entry *dst)
-+{
-+ BUG_ON(c3cn->cdev != cdev);
-+ c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs;
-+ c3cn->wr_unacked = 0;
-+ c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
-+
-+ c3cn->ctrl_skb_cache = alloc_skb(CTRL_SKB_LEN, gfp_any());
-+ reset_wr_list(c3cn);
-+}
-+
-+static void act_open_retry_timer(unsigned long data)
-+{
-+ struct sk_buff *skb;
-+ struct s3_conn *c3cn = (struct s3_conn *)data;
-+
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
-+
-+ spin_lock(&c3cn->lock);
-+ skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_ATOMIC);
-+ if (!skb)
-+ fail_act_open(c3cn, ENOMEM);
-+ else {
-+ skb->sk = (struct sock *)c3cn;
-+ set_arp_failure_handler(skb, act_open_req_arp_failure);
-+ mk_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
-+ l2t_send(c3cn->cdev, skb, c3cn->l2t);
-+ }
-+ spin_unlock(&c3cn->lock);
-+ c3cn_put(c3cn);
-+}
-+
-+/*
-+ * Convert an ACT_OPEN_RPL status to a Linux errno.
-+ */
-+static int act_open_rpl_status_to_errno(int status)
-+{
-+ switch (status) {
-+ case CPL_ERR_CONN_RESET:
-+ return ECONNREFUSED;
-+ case CPL_ERR_ARP_MISS:
-+ return EHOSTUNREACH;
-+ case CPL_ERR_CONN_TIMEDOUT:
-+ return ETIMEDOUT;
-+ case CPL_ERR_TCAM_FULL:
-+ return ENOMEM;
-+ case CPL_ERR_CONN_EXIST:
-+ printk(KERN_ERR "ACTIVE_OPEN_RPL: 4-tuple in use\n");
-+ return EADDRINUSE;
-+ default:
-+ return EIO;
-+ }
-+}
-+
-+/*
-+ * Convert the status code of an ABORT_REQ into a Linux error code. Also
-+ * indicate whether RST should be sent in response.
-+ */
-+static int abort_status_to_errno(struct s3_conn *c3cn,
-+ int abort_reason, int *need_rst)
-+{
-+ switch (abort_reason) {
-+ case CPL_ERR_BAD_SYN: /* fall through */
-+ case CPL_ERR_CONN_RESET:
-+ return c3cn->state == C3CN_STATE_CLOSING ? EPIPE : ECONNRESET;
-+ case CPL_ERR_XMIT_TIMEDOUT:
-+ case CPL_ERR_PERSIST_TIMEDOUT:
-+ case CPL_ERR_FINWAIT2_TIMEDOUT:
-+ case CPL_ERR_KEEPALIVE_TIMEDOUT:
-+ return ETIMEDOUT;
-+ default:
-+ return EIO;
-+ }
-+}
-+
-+static void send_abort_rpl(struct sk_buff *skb, struct t3cdev *cdev,
-+ int rst_status)
-+{
-+ struct sk_buff *reply_skb;
-+ struct cpl_abort_req_rss *req = cplhdr(skb);
-+
-+ reply_skb = get_cpl_reply_skb(skb, sizeof(struct cpl_abort_rpl),
-+ gfp_any());
-+
-+ reply_skb->priority = CPL_PRIORITY_DATA;
-+ set_abort_rpl_wr(reply_skb, GET_TID(req), rst_status);
-+ kfree_skb(skb);
-+ cxgb3_ofld_send(cdev, reply_skb);
-+}
-+
-+/*
-+ * Returns an sk_buff for a reply CPL message of size len. If the input
-+ * sk_buff has no other users it is trimmed and reused, otherwise a new buffer
-+ * is allocated. The input skb must be of size at least len. Note that this
-+ * operation does not destroy the original skb data even if it decides to reuse
-+ * the buffer.
-+ */
-+static struct sk_buff *get_cpl_reply_skb(struct sk_buff *skb, size_t len,
-+ gfp_t gfp)
-+{
-+ if (likely(!skb_cloned(skb))) {
-+ BUG_ON(skb->len < len);
-+ __skb_trim(skb, len);
-+ skb_get(skb);
-+ } else {
-+ skb = alloc_skb(len, gfp);
-+ if (skb)
-+ __skb_put(skb, len);
-+ }
-+ return skb;
-+}
-+
-+/*
-+ * Release resources held by an offload connection (TID, L2T entry, etc.)
-+ */
-+static void t3_release_offload_resources(struct s3_conn *c3cn)
-+{
-+ struct t3cdev *cdev = c3cn->cdev;
-+ unsigned int tid = c3cn->tid;
-+
-+ if (!cdev)
-+ return;
-+
-+ c3cn->qset = 0;
-+
-+ kfree_skb(c3cn->ctrl_skb_cache);
-+ c3cn->ctrl_skb_cache = NULL;
-+
-+ if (c3cn->wr_avail != c3cn->wr_max) {
-+ purge_wr_queue(c3cn);
-+ reset_wr_list(c3cn);
-+ }
-+
-+ if (c3cn->l2t) {
-+ l2t_release(L2DATA(cdev), c3cn->l2t);
-+ c3cn->l2t = NULL;
-+ }
-+
-+ if (c3cn->state == C3CN_STATE_SYN_SENT) /* we have ATID */
-+ free_atid(cdev, tid);
-+ else { /* we have TID */
-+ cxgb3_remove_tid(cdev, (void *)c3cn, tid);
-+ c3cn_put(c3cn);
-+ }
-+
-+ c3cn->cdev = NULL;
-+}
-+
-+/*
-+ * Handles Rx data that arrives in a state where the connection isn't
-+ * accepting new data.
-+ */
-+static void handle_excess_rx(struct s3_conn *c3cn, struct sk_buff *skb)
-+{
-+ if (!c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN))
-+ abort_conn(c3cn, skb);
-+
-+ kfree_skb(skb);
-+}
-+
-+/*
-+ * Like get_cpl_reply_skb() but the returned buffer starts out empty.
-+ */
-+static struct sk_buff *__get_cpl_reply_skb(struct sk_buff *skb, size_t len,
-+ gfp_t gfp)
-+{
-+ if (likely(!skb_cloned(skb) && !skb->data_len)) {
-+ __skb_trim(skb, 0);
-+ skb_get(skb);
-+ } else
-+ skb = alloc_skb(len, gfp);
-+ return skb;
-+}
-+
-+/*
-+ * Completes some final bits of initialization for just established connections
-+ * and changes their state to C3CN_STATE_ESTABLISHED.
-+ *
-+ * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
-+ */
-+static void make_established(struct s3_conn *c3cn, u32 snd_isn,
-+ unsigned int opt)
-+{
-+ c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
-+
-+ c3cn->write_seq = c3cn->snd_nxt = c3cn->snd_una = snd_isn;
-+
-+ /*
-+ * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
-+ * pass through opt0.
-+ */
-+ if (cxgb3_rcv_win > (M_RCV_BUFSIZ << 10))
-+ c3cn->rcv_wup -= cxgb3_rcv_win - (M_RCV_BUFSIZ << 10);
-+
-+ dst_confirm(c3cn->dst_cache);
-+
-+ smp_mb();
-+ c3cn_set_state(c3cn, C3CN_STATE_ESTABLISHED);
-+}
---- /dev/null
-+++ b/drivers/scsi/cxgb3i/cxgb3i_offload.h
-@@ -0,0 +1,220 @@
-+/*
-+ * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
-+ *
-+ * Written by Dimitris Michailidis (dm@chelsio.com)
-+ *
-+ * This program is distributed in the hope that it will be useful, but WITHOUT
-+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-+ * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
-+ * release for licensing terms and conditions.
-+ */
-+
-+#ifndef _CXGB3I_OFFLOAD_H
-+#define _CXGB3I_OFFLOAD_H
-+
-+#include <linux/skbuff.h>
-+#include <net/tcp.h>
-+
-+#include "common.h"
-+#include "adapter.h"
-+#include "t3cdev.h"
-+#include "cxgb3_offload.h"
-+
-+#define cxgb3i_log_error(fmt...) printk(KERN_ERR "cxgb3i: ERR! " fmt)
-+#define cxgb3i_log_warn(fmt...) printk(KERN_WARNING "cxgb3i: WARN! " fmt)
-+#define cxgb3i_log_info(fmt...) printk(KERN_INFO "cxgb3i: " fmt)
-+
-+#ifdef __DEBUG_CXGB3I__
-+#define cxgb3i_log_debug(fmt, args...) \
-+ printk(KERN_INFO "cxgb3i: %s - " fmt, __func__ , ## args)
-+#else
-+#define cxgb3i_log_debug(fmt...)
-+#endif
-+
-+#ifdef __DEBUG_C3CN_CONN__
-+#define c3cn_conn_debug cxgb3i_log_debug
-+#else
-+#define c3cn_conn_debug(fmt...)
-+#endif
-+
-+/*
-+ * Data structure to keep track of cxgb3 connection.
-+ */
-+struct s3_conn {
-+ struct net_device *dev; /* net device of with connection */
-+ struct t3cdev *cdev; /* adapter t3cdev for net device */
-+ unsigned long flags; /* see c3cn_flags below */
-+ int tid; /* ID of TCP Control Block */
-+ int qset; /* queue Set used by connection */
-+ int mss_idx; /* Maximum Segment Size table index */
-+ struct l2t_entry *l2t; /* ARP resolution for offload packets */
-+ int wr_max; /* maximum in-flight writes */
-+ int wr_avail; /* number of writes available */
-+ int wr_unacked; /* writes since last request for */
-+ /* completion notification */
-+ struct sk_buff *wr_pending_head;/* head of pending write queue */
-+ struct sk_buff *wr_pending_tail;/* tail of pending write queue */
-+ struct sk_buff *ctrl_skb_cache; /* single entry cached skb for */
-+ /* short-term control operations */
-+ spinlock_t lock; /* connection status lock */
-+ atomic_t refcnt; /* reference count on connection */
-+ volatile unsigned int state; /* connection state */
-+ struct sockaddr_in saddr; /* source IP/port address */
-+ struct sockaddr_in daddr; /* destination IP/port address */
-+ struct dst_entry *dst_cache; /* reference to destination route */
-+ unsigned char shutdown; /* shutdown status */
-+ struct sk_buff_head receive_queue;/* received PDUs */
-+ struct sk_buff_head write_queue;/* un-pushed pending writes */
-+
-+ struct timer_list retry_timer; /* retry timer for various operations */
-+ int err; /* connection error status */
-+ rwlock_t callback_lock; /* lock for opaque user context */
-+ void *user_data; /* opaque user context */
-+
-+ u32 rcv_nxt; /* what we want to receive next */
-+ u32 copied_seq; /* head of yet unread data */
-+ u32 rcv_wup; /* rcv_nxt on last window update sent */
-+ u32 snd_nxt; /* next sequence we send */
-+ u32 snd_una; /* first byte we want an ack for */
-+
-+ u32 write_seq; /* tail+1 of data held in send buffer */
-+};
-+
-+/* Flags in c3cn->shutdown */
-+#define C3CN_RCV_SHUTDOWN 0x1
-+#define C3CN_SEND_SHUTDOWN 0x2
-+#define C3CN_SHUTDOWN_MASK (C3CN_RCV_SHUTDOWN | C3CN_SEND_SHUTDOWN)
-+
-+/*
-+ * connection state bitmap
-+ */
-+#define C3CN_STATE_CLOSE 0x1
-+#define C3CN_STATE_SYN_SENT 0x2
-+#define C3CN_STATE_ESTABLISHED 0x4
-+#define C3CN_STATE_CLOSING 0x8
-+#define C3CN_STATE_ABORING 0x10
-+
-+#define C3CN_STATE_MASK 0xFF
-+
-+static inline unsigned int c3cn_in_state(const struct s3_conn *c3cn,
-+ unsigned int states)
-+{
-+ return states & c3cn->state;
-+}
-+
-+/*
-+ * Connection flags -- many to track some close related events.
-+ */
-+enum c3cn_flags {
-+ C3CN_ABORT_RPL_RCVD, /* received one ABORT_RPL_RSS message */
-+ C3CN_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
-+ C3CN_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
-+ C3CN_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
-+
-+ C3CN_ABORT_RPL_PENDING, /* expecting an abort reply */
-+ C3CN_CLOSE_CON_REQUESTED, /* we've sent a close_conn_req */
-+ C3CN_TX_DATA_SENT, /* already sent a TX_DATA WR */
-+ C3CN_CLOSE_NEEDED, /* need to be closed */
-+ C3CN_DONE,
-+};
-+
-+/*
-+ * Per adapter data. Linked off of each Ethernet device port on the adapter.
-+ * Also available via the t3cdev structure since we have pointers to our port
-+ * net_device's there ...
-+ */
-+struct cxgb3i_sdev_data {
-+ struct list_head list; /* links for list of all adapters */
-+ struct t3cdev *cdev; /* adapter t3cdev */
-+ struct cxgb3_client *client; /* CPL client pointer */
-+ struct adap_ports *ports; /* array of adapter ports */
-+ unsigned int rx_page_size; /* RX page size */
-+ struct sk_buff_head deferq; /* queue for processing replies from */
-+ /* worker thread context */
-+ struct work_struct deferq_task; /* worker thread */
-+};
-+#define NDEV2CDATA(ndev) (*(struct cxgb3i_sdev_data **)&(ndev)->ec_ptr)
-+#define CXGB3_SDEV_DATA(cdev) NDEV2CDATA((cdev)->lldev)
-+
-+/*
-+ * Primary API routines.
-+ */
-+void cxgb3i_sdev_cleanup(void);
-+int cxgb3i_sdev_init(cxgb3_cpl_handler_func *);
-+void cxgb3i_sdev_add(struct t3cdev *, struct cxgb3_client *);
-+void cxgb3i_sdev_remove(struct t3cdev *);
-+
-+struct s3_conn *cxgb3i_c3cn_create(void);
-+int cxgb3i_c3cn_connect(struct s3_conn *, struct sockaddr_in *);
-+void cxgb3i_c3cn_rx_credits(struct s3_conn *, int);
-+int cxgb3i_c3cn_send_pdus(struct s3_conn *, struct sk_buff *, int);
-+void cxgb3i_c3cn_release(struct s3_conn *);
-+
-+/*
-+ * Definitions for sk_buff state and ULP mode management.
-+ */
-+
-+struct cxgb3_skb_cb {
-+ __u8 flags; /* see C3CB_FLAG_* below */
-+ __u8 ulp_mode; /* ULP mode/submode of sk_buff */
-+ __u32 seq; /* sequence number */
-+ __u32 ddigest; /* ULP rx_data_ddp selected field */
-+ __u32 pdulen; /* ULP rx_data_ddp selected field */
-+ __u8 ulp_data[16]; /* scratch area for ULP */
-+};
-+
-+#define CXGB3_SKB_CB(skb) ((struct cxgb3_skb_cb *)&((skb)->cb[0]))
-+
-+#define skb_ulp_mode(skb) (CXGB3_SKB_CB(skb)->ulp_mode)
-+#define skb_ulp_ddigest(skb) (CXGB3_SKB_CB(skb)->ddigest)
-+#define skb_ulp_pdulen(skb) (CXGB3_SKB_CB(skb)->pdulen)
-+#define skb_ulp_data(skb) (CXGB3_SKB_CB(skb)->ulp_data)
-+
-+enum {
-+ C3CB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
-+ C3CB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
-+ C3CB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */
-+ C3CB_FLAG_COMPL = 1 << 4, /* request WR completion */
-+};
-+
-+/*
-+ * Top-level CPL message processing used by most CPL messages that
-+ * pertain to connections.
-+ */
-+static inline void process_cpl_msg(void (*fn)(struct s3_conn *,
-+ struct sk_buff *),
-+ struct s3_conn *c3cn,
-+ struct sk_buff *skb)
-+{
-+ spin_lock(&c3cn->lock);
-+ fn(c3cn, skb);
-+ spin_unlock(&c3cn->lock);
-+}
-+
-+/*
-+ * Opaque version of structure the SGE stores at skb->head of TX_DATA packets
-+ * and for which we must reserve space.
-+ */
-+struct sge_opaque_hdr {
-+ void *dev;
-+ dma_addr_t addr[MAX_SKB_FRAGS + 1];
-+};
-+
-+/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
-+#define TX_HEADER_LEN \
-+ (sizeof(struct tx_data_wr) + sizeof(struct sge_opaque_hdr))
-+
-+void *cxgb3i_alloc_big_mem(unsigned int);
-+void cxgb3i_free_big_mem(void *);
-+
-+/*
-+ * get and set private ip for iscsi traffic
-+ */
-+#define cxgb3i_get_private_ipv4addr(ndev) \
-+ (((struct port_info *)(netdev_priv(ndev)))->iscsi_ipv4addr)
-+#define cxgb3i_set_private_ipv4addr(ndev, addr) \
-+ (((struct port_info *)(netdev_priv(ndev)))->iscsi_ipv4addr) = addr
-+
-+/* max. connections per adapter */
-+#define CXGB3I_MAX_CONN 16384
-+#endif /* _CXGB3_OFFLOAD_H */
---- /dev/null
-+++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
-@@ -0,0 +1,741 @@
-+/*
-+ * cxgb3i_ulp2.c: Chelsio S3xx iSCSI driver.
-+ *
-+ * Copyright (c) 2008 Chelsio Communications, Inc.
-+ *
-+ * This program is free software; you can redistribute it and/or modify
-+ * it under the terms of the GNU General Public License as published by
-+ * the Free Software Foundation.
-+ *
-+ * Written by: Karen Xie (kxie@chelsio.com)
-+ */
-+
-+#include <linux/skbuff.h>
-+#include <scsi/scsi_cmnd.h>
-+#include <scsi/scsi_host.h>
-+#include <linux/crypto.h>
-+
-+#include "cxgb3i.h"
-+#include "cxgb3i_ulp2.h"
-+
-+#ifdef __DEBUG_CXGB3I_RX__
-+#define cxgb3i_rx_debug cxgb3i_log_debug
-+#else
-+#define cxgb3i_rx_debug(fmt...)
-+#endif
-+
-+#ifdef __DEBUG_CXGB3I_TX__
-+#define cxgb3i_tx_debug cxgb3i_log_debug
-+#else
-+#define cxgb3i_tx_debug(fmt...)
-+#endif
-+
-+#ifdef __DEBUG_CXGB3I_TAG__
-+#define cxgb3i_tag_debug cxgb3i_log_debug
-+#else
-+#define cxgb3i_tag_debug(fmt...)
-+#endif
-+
-+#ifdef __DEBUG_CXGB3I_DDP__
-+#define cxgb3i_ddp_debug cxgb3i_log_debug
-+#else
-+#define cxgb3i_ddp_debug(fmt...)
-+#endif
-+
-+static struct page *pad_page;
-+
-+#define ULP2_PGIDX_MAX 4
-+#define ULP2_4K_PAGE_SHIFT 12
-+#define ULP2_4K_PAGE_MASK (~((1UL << ULP2_4K_PAGE_SHIFT) - 1))
-+static unsigned char ddp_page_order[ULP2_PGIDX_MAX];
-+static unsigned long ddp_page_size[ULP2_PGIDX_MAX];
-+static unsigned char ddp_page_shift[ULP2_PGIDX_MAX];
-+static unsigned char sw_tag_idx_bits;
-+static unsigned char sw_tag_age_bits;
-+
-+static void cxgb3i_ddp_page_init(void)
-+{
-+ int i;
-+ unsigned long n = PAGE_SIZE >> ULP2_4K_PAGE_SHIFT;
-+
-+ if (PAGE_SIZE & (~ULP2_4K_PAGE_MASK)) {
-+ cxgb3i_log_debug("PAGE_SIZE 0x%lx is not multiple of 4K, "
-+ "ddp disabled.\n", PAGE_SIZE);
-+ return;
-+ }
-+ n = __ilog2_u32(n);
-+ for (i = 0; i < ULP2_PGIDX_MAX; i++, n++) {
-+ ddp_page_order[i] = n;
-+ ddp_page_shift[i] = ULP2_4K_PAGE_SHIFT + n;
-+ ddp_page_size[i] = 1 << ddp_page_shift[i];
-+ cxgb3i_log_debug("%d, order %u, shift %u, size 0x%lx.\n", i,
-+ ddp_page_order[i], ddp_page_shift[i],
-+ ddp_page_size[i]);
-+ }
-+
-+ sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
-+ sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
-+}
-+
-+static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
-+{
-+ struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
-+
-+ req->wr.wr_lo = 0;
-+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
-+ req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
-+ V_ULPTX_CMD(ULP_MEM_WRITE));
-+ req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) |
-+ V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1));
-+}
-+
-+static int set_ddp_map(struct cxgb3i_adapter *snic, struct pagepod_hdr *hdr,
-+ unsigned int idx, unsigned int npods,
-+ struct scatterlist *sgl, unsigned int sgcnt)
-+{
-+ struct cxgb3i_ddp_info *ddp = &snic->ddp;
-+ struct scatterlist *sg = sgl;
-+ unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
-+ int i;
-+
-+ for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
-+ struct sk_buff *skb;
-+ struct pagepod *ppod;
-+ int j, k;
-+ skb =
-+ alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
-+ GFP_ATOMIC);
-+ if (!skb) {
-+ cxgb3i_log_debug("skb OMM.\n");
-+ return -ENOMEM;
-+ }
-+ skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
-+
-+ ulp_mem_io_set_hdr(skb, pm_addr);
-+ ppod =
-+ (struct pagepod *)(skb->head + sizeof(struct ulp_mem_io));
-+ memcpy(&(ppod->hdr), hdr, sizeof(struct pagepod));
-+ for (j = 0, k = i * 4; j < 5; j++, k++) {
-+ if (k < sgcnt) {
-+ ppod->addr[j] = cpu_to_be64(sg_dma_address(sg));
-+ if (j < 4)
-+ sg = sg_next(sg);
-+ } else
-+ ppod->addr[j] = 0UL;
-+ }
-+
-+ skb->priority = CPL_PRIORITY_CONTROL;
-+ cxgb3_ofld_send(snic->tdev, skb);
-+ }
-+ return 0;
-+}
-+
-+static int clear_ddp_map(struct cxgb3i_adapter *snic, unsigned int idx,
-+ unsigned int npods)
-+{
-+ struct cxgb3i_ddp_info *ddp = &snic->ddp;
-+ unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
-+ int i;
-+
-+ for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
-+ struct sk_buff *skb;
-+ skb =
-+ alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
-+ GFP_ATOMIC);
-+ if (!skb)
-+ return -ENOMEM;
-+ skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
-+ memset((skb->head + sizeof(struct ulp_mem_io)), 0, PPOD_SIZE);
-+ ulp_mem_io_set_hdr(skb, pm_addr);
-+ skb->priority = CPL_PRIORITY_CONTROL;
-+ cxgb3_ofld_send(snic->tdev, skb);
-+ }
-+ return 0;
-+}
-+
-+static int cxgb3i_ddp_sgl_check(struct scatterlist *sgl, unsigned int sgcnt)
-+{
-+ struct scatterlist *sg;
-+ int i;
-+
-+ /* make sure the sgl is fit for ddp:
-+ * each has the same page size, and
-+ * first & last page do not need to be used completely, and
-+ * the rest of page must be used completely
-+ */
-+ for_each_sg(sgl, sg, sgcnt, i) {
-+ if ((i && sg->offset) ||
-+ ((i != sgcnt - 1) &&
-+ (sg->length + sg->offset) != PAGE_SIZE)) {
-+ cxgb3i_tag_debug("sg %u/%u, off %u, len %u.\n",
-+ i, sgcnt, sg->offset, sg->length);
-+ return -EINVAL;
-+ }
-+ }
-+
-+ return 0;
-+}
-+
-+static inline int ddp_find_unused_entries(struct cxgb3i_ddp_info *ddp,
-+ int start, int max, int count)
-+{
-+ unsigned int i, j;
-+
-+ spin_lock(&ddp->map_lock);
-+ for (i = start; i <= max;) {
-+ for (j = 0; j < count; j++) {
-+ if (ddp->map[i + j])
-+ break;
-+ }
-+ if (j == count) {
-+ memset(&ddp->map[i], 1, count);
-+ spin_unlock(&ddp->map_lock);
-+ return i;
-+ }
-+ i += j + 1;
-+ }
-+ spin_unlock(&ddp->map_lock);
-+ return -EBUSY;
-+}
-+
-+static inline void ddp_unmark_entries(struct cxgb3i_ddp_info *ddp,
-+ int start, int count)
-+{
-+ spin_lock(&ddp->map_lock);
-+ memset(&ddp->map[start], 0, count);
-+ spin_unlock(&ddp->map_lock);
-+}
-+
-+static inline int sgl_map(struct cxgb3i_adapter *snic,
-+ struct scatterlist *sgl, unsigned int sgcnt)
-+{
-+ struct scatterlist *sg;
-+ int i, err;
-+
-+ for_each_sg(sgl, sg, sgcnt, i) {
-+ err = pci_map_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
-+ if (err <= 0) {
-+ cxgb3i_tag_debug("sgcnt %d/%u, pci map failed %d.\n",
-+ i, sgcnt, err);
-+ return err;
-+ }
-+ }
-+ return sgcnt;
-+}
-+
-+static inline void sgl_unmap(struct cxgb3i_adapter *snic,
-+ struct scatterlist *sgl, unsigned int sgcnt)
-+{
-+ struct scatterlist *sg;
-+ int i;
-+
-+ for_each_sg(sgl, sg, sgcnt, i) {
-+ if (sg_dma_address(sg))
-+ pci_unmap_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
-+ else
-+ break;
-+ }
-+}
-+
-+u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *snic, unsigned int tid,
-+ u32 sw_tag, unsigned int xferlen,
-+ struct scatterlist *sgl, unsigned int sgcnt)
-+{
-+ struct cxgb3i_ddp_info *ddp = &snic->ddp;
-+ struct pagepod_hdr hdr;
-+ unsigned int npods;
-+ int idx = -1, idx_max;
-+ u32 tag;
-+ int err;
-+
-+ if (!ddp || !sgcnt || xferlen < PAGE_SIZE) {
-+ cxgb3i_tag_debug("sgcnt %u, xferlen %u < %lu, NO DDP.\n",
-+ sgcnt, xferlen, PAGE_SIZE);
-+ return RESERVED_ITT;
-+ }
-+
-+ err = cxgb3i_ddp_sgl_check(sgl, sgcnt);
-+ if (err < 0) {
-+ cxgb3i_tag_debug("sgcnt %u, xferlen %u, SGL check fail.\n",
-+ sgcnt, xferlen);
-+ return RESERVED_ITT;
-+ }
-+
-+ npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
-+ idx_max = ddp->nppods - npods + 1;
-+
-+ if (ddp->idx_last == ddp->nppods)
-+ idx = ddp_find_unused_entries(ddp, 0, idx_max, npods);
-+ else {
-+ idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1, idx_max,
-+ npods);
-+ if ((idx < 0) && (ddp->idx_last >= npods))
-+ idx = ddp_find_unused_entries(ddp, 0,
-+ ddp->idx_last - npods + 1,
-+ npods);
-+ }
-+ if (idx < 0) {
-+ cxgb3i_tag_debug("sgcnt %u, xferlen %u, npods %u NO DDP.\n",
-+ sgcnt, xferlen, npods);
-+ return RESERVED_ITT;
-+ }
-+
-+ err = sgl_map(snic, sgl, sgcnt);
-+ if (err < sgcnt)
-+ goto unmap_sgl;
-+
-+ tag = sw_tag | (idx << snic->tag_format.rsvd_shift);
-+
-+ hdr.rsvd = 0;
-+ hdr.vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid));
-+ hdr.pgsz_tag_clr = htonl(tag & snic->tag_format.rsvd_tag_mask);
-+ hdr.maxoffset = htonl(xferlen);
-+ hdr.pgoffset = htonl(sgl->offset);
-+
-+ if (set_ddp_map(snic, &hdr, idx, npods, sgl, sgcnt) < 0)
-+ goto unmap_sgl;
-+
-+ ddp->idx_last = idx;
-+ cxgb3i_tag_debug("tid 0x%x, xfer %u, 0x%x -> ddp 0x%x (0x%x, %u).\n",
-+ tid, xferlen, sw_tag, tag, idx, npods);
-+ return tag;
-+
-+unmap_sgl:
-+ sgl_unmap(snic, sgl, sgcnt);
-+ ddp_unmark_entries(ddp, idx, npods);
-+ return RESERVED_ITT;
-+}
-+
-+void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *snic, u32 tag,
-+ struct scatterlist *sgl, unsigned int sgcnt)
-+{
-+ u32 idx = (tag >> snic->tag_format.rsvd_shift) &
-+ snic->tag_format.rsvd_mask;
-+ unsigned int npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
-+
-+ if (idx < snic->tag_format.rsvd_mask) {
-+ cxgb3i_tag_debug("ddp tag 0x%x, release idx 0x%x, npods %u.\n",
-+ tag, idx, npods);
-+ clear_ddp_map(snic, idx, npods);
-+ ddp_unmark_entries(&snic->ddp, idx, npods);
-+ sgl_unmap(snic, sgl, sgcnt);
-+ }
-+}
-+
-+int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *cconn, int hcrc, int dcrc)
-+{
-+ struct iscsi_tcp_conn *tcp_conn = cconn->conn->dd_data;
-+ struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
-+ struct sk_buff *skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
-+ GFP_KERNEL | __GFP_NOFAIL);
-+ struct cpl_set_tcb_field *req;
-+ u32 submode = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
-+
-+ /* set up ulp submode and page size */
-+ req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
-+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
-+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, c3cn->tid));
-+ req->reply = V_NO_REPLY(1);
-+ req->cpu_idx = 0;
-+ req->word = htons(31);
-+ req->mask = cpu_to_be64(0xFF000000);
-+ /* the connection page size is always the same as ddp-pgsz0 */
-+ req->val = cpu_to_be64(submode << 24);
-+ skb->priority = CPL_PRIORITY_CONTROL;
-+
-+ cxgb3_ofld_send(c3cn->cdev, skb);
-+ return 0;
-+}
-+
-+static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn,
-+ struct sk_buff *skb)
-+{
-+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
-+ struct iscsi_segment *segment = &tcp_conn->in.segment;
-+ struct iscsi_hdr *hdr = (struct iscsi_hdr *)tcp_conn->in.hdr_buf;
-+ unsigned char *buf = (unsigned char *)hdr;
-+ unsigned int offset = sizeof(struct iscsi_hdr);
-+ int err;
-+
-+ cxgb3i_rx_debug("conn 0x%p, skb 0x%p, len %u, flag 0x%x.\n",
-+ conn, skb, skb->len, skb_ulp_mode(skb));
-+
-+ /* read bhs */
-+ err = skb_copy_bits(skb, 0, buf, sizeof(struct iscsi_hdr));
-+ if (err < 0)
-+ return err;
-+ segment->copied = sizeof(struct iscsi_hdr);
-+ /* read ahs */
-+ if (hdr->hlength) {
-+ unsigned int ahslen = hdr->hlength << 2;
-+ /* Make sure we don't overflow */
-+ if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf))
-+ return -ISCSI_ERR_AHSLEN;
-+ err = skb_copy_bits(skb, offset, buf + offset, ahslen);
-+ if (err < 0)
-+ return err;
-+ offset += ahslen;
-+ }
-+ /* header digest */
-+ if (conn->hdrdgst_en)
-+ offset += ISCSI_DIGEST_SIZE;
-+
-+ /* check header digest */
-+ segment->status = (conn->hdrdgst_en &&
-+ (skb_ulp_mode(skb) & ULP2_FLAG_HCRC_ERROR)) ?
-+ ISCSI_SEGMENT_DGST_ERR : 0;
-+
-+ hdr->itt = ntohl(hdr->itt);
-+ segment->total_copied = segment->total_size;
-+ tcp_conn->in.hdr = hdr;
-+ err = iscsi_tcp_hdr_dissect(conn, hdr);
-+ if (err)
-+ return err;
-+
-+ if (tcp_conn->in.datalen) {
-+ segment = &tcp_conn->in.segment;
-+ segment->status = (conn->datadgst_en &&
-+ (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) ?
-+ ISCSI_SEGMENT_DGST_ERR : 0;
-+ if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
-+ cxgb3i_ddp_debug("opcode 0x%x, data %u, ddp'ed.\n",
-+ hdr->opcode & ISCSI_OPCODE_MASK,
-+ tcp_conn->in.datalen);
-+ segment->total_copied = segment->total_size;
-+ } else
-+ offset += sizeof(struct cpl_iscsi_hdr_norss);
-+
-+ while (segment->total_copied < segment->total_size) {
-+ iscsi_tcp_segment_map(segment, 1);
-+ err = skb_copy_bits(skb, offset, segment->data,
-+ segment->size);
-+ iscsi_tcp_segment_unmap(segment);
-+ if (err)
-+ return err;
-+ segment->total_copied += segment->size;
-+ offset += segment->size;
-+
-+ if (segment->total_copied < segment->total_size)
-+ iscsi_tcp_segment_init_sg(segment,
-+ sg_next(segment->sg),
-+ 0);
-+ }
-+ err = segment->done(tcp_conn, segment);
-+ }
-+ return err;
-+}
-+
-+static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
-+{
-+ u8 submode = 0;
-+
-+ if (hcrc)
-+ submode |= 1;
-+ if (dcrc)
-+ submode |= 2;
-+ skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) | submode;
-+}
-+
-+int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *conn)
-+{
-+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
-+ struct iscsi_segment *hdr_seg = &tcp_conn->out.segment;
-+ struct iscsi_segment *data_seg = &tcp_conn->out.data_segment;
-+ unsigned int hdrlen = hdr_seg->total_size;
-+ unsigned int datalen = data_seg->total_size;
-+ unsigned int padlen = iscsi_padding(datalen);
-+ unsigned int copymax = SKB_MAX_HEAD(TX_HEADER_LEN);
-+ unsigned int copylen;
-+ struct sk_buff *skb;
-+ unsigned char *dst;
-+ int err = -EAGAIN;
-+
-+ if (data_seg->data && ((datalen + padlen) < copymax))
-+ copylen = hdrlen + datalen + padlen;
-+ else
-+ copylen = hdrlen;
-+
-+ /* supports max. 16K pdus, so one skb is enough to hold all the data */
-+ skb = alloc_skb(TX_HEADER_LEN + copylen, GFP_ATOMIC);
-+ if (!skb)
-+ return -EAGAIN;
-+
-+ skb_reserve(skb, TX_HEADER_LEN);
-+ skb_put(skb, copylen);
-+ dst = skb->data;
-+
-+ tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
-+
-+ memcpy(dst, hdr_seg->data, hdrlen);
-+ dst += hdrlen;
-+
-+ if (!datalen)
-+ goto send_pdu;
-+
-+ if (data_seg->data) {
-+ /* data is in a linear buffer */
-+ if (copylen > hdrlen) {
-+ /* data fits in the skb's headroom */
-+ memcpy(dst, data_seg->data, datalen);
-+ dst += datalen;
-+ if (padlen)
-+ memset(dst, 0, padlen);
-+ } else {
-+ unsigned int offset = 0;
-+ while (datalen) {
-+ struct page *page = alloc_page(GFP_ATOMIC);
-+ int idx = skb_shinfo(skb)->nr_frags;
-+ skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
-+
-+ if (!page)
-+ goto free_skb;
-+
-+ frag->page = page;
-+ frag->page_offset = 0;
-+ if (datalen > PAGE_SIZE)
-+ frag->size = PAGE_SIZE;
-+ else
-+ frag->size = datalen;
-+ memcpy(page_address(page),
-+ data_seg->data + offset, frag->size);
-+
-+ skb_shinfo(skb)->nr_frags++;
-+ datalen -= frag->size;
-+ offset += frag->size;
-+ }
-+ }
-+ } else {
-+ struct scatterlist *sg = data_seg->sg;
-+ unsigned int offset = data_seg->sg_offset;
-+ while (datalen) {
-+ int idx = skb_shinfo(skb)->nr_frags;
-+ skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
-+ struct page *pg = sg_page(sg);
-+
-+ get_page(pg);
-+ frag->page = pg;
-+ frag->page_offset = offset + sg->offset;
-+ frag->size = min(sg->length, datalen);
-+
-+ offset = 0;
-+ skb_shinfo(skb)->nr_frags++;
-+ datalen -= frag->size;
-+ sg = sg_next(sg);
-+ }
-+ }
-+
-+ if (skb_shinfo(skb)->nr_frags) {
-+ if (padlen) {
-+ int idx = skb_shinfo(skb)->nr_frags;
-+ skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
-+ frag->page = pad_page;
-+ frag->page_offset = 0;
-+ frag->size = padlen;
-+ skb_shinfo(skb)->nr_frags++;
-+ }
-+ datalen = data_seg->total_size + padlen;
-+ skb->data_len += datalen;
-+ skb->truesize += datalen;
-+ skb->len += datalen;
-+ }
-+
-+send_pdu:
-+ err = cxgb3i_c3cn_send_pdus((struct s3_conn *)tcp_conn->sock,
-+ skb, MSG_DONTWAIT | MSG_NOSIGNAL);
-+ if (err > 0) {
-+ int pdulen = hdrlen + datalen + padlen;
-+ if (conn->hdrdgst_en)
-+ pdulen += ISCSI_DIGEST_SIZE;
-+ if (datalen && conn->datadgst_en)
-+ pdulen += ISCSI_DIGEST_SIZE;
-+
-+ hdr_seg->total_copied = hdr_seg->total_size;
-+ if (datalen)
-+ data_seg->total_copied = data_seg->total_size;
-+ conn->txdata_octets += pdulen;
-+ return pdulen;
-+ }
-+
-+free_skb:
-+ kfree_skb(skb);
-+ if (err < 0 && err != -EAGAIN) {
-+ cxgb3i_log_error("conn 0x%p, xmit err %d.\n", conn, err);
-+ iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
-+ return err;
-+ }
-+ return -EAGAIN;
-+}
-+
-+int cxgb3i_ulp2_init(void)
-+{
-+ pad_page = alloc_page(GFP_KERNEL);
-+ if (!pad_page)
-+ return -ENOMEM;
-+ memset(page_address(pad_page), 0, PAGE_SIZE);
-+ cxgb3i_ddp_page_init();
-+ return 0;
-+}
-+
-+void cxgb3i_ulp2_cleanup(void)
-+{
-+ if (pad_page) {
-+ __free_page(pad_page);
-+ pad_page = NULL;
-+ }
-+}
-+
-+void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn)
-+{
-+ struct sk_buff *skb;
-+ unsigned int read = 0;
-+ struct iscsi_conn *conn = c3cn->user_data;
-+ int err = 0;
-+
-+ cxgb3i_rx_debug("cn 0x%p.\n", c3cn);
-+
-+ read_lock(&c3cn->callback_lock);
-+ if (unlikely(!conn || conn->suspend_rx)) {
-+ cxgb3i_rx_debug("conn 0x%p, id %d, suspend_rx %lu!\n",
-+ conn, conn ? conn->id : 0xFF,
-+ conn ? conn->suspend_rx : 0xFF);
-+ read_unlock(&c3cn->callback_lock);
-+ return;
-+ }
-+ skb = skb_peek(&c3cn->receive_queue);
-+ while (!err && skb) {
-+ __skb_unlink(skb, &c3cn->receive_queue);
-+ read += skb_ulp_pdulen(skb);
-+ err = cxgb3i_conn_read_pdu_skb(conn, skb);
-+ __kfree_skb(skb);
-+ skb = skb_peek(&c3cn->receive_queue);
-+ }
-+ read_unlock(&c3cn->callback_lock);
-+ if (c3cn) {
-+ c3cn->copied_seq += read;
-+ cxgb3i_c3cn_rx_credits(c3cn, read);
-+ }
-+ conn->rxdata_octets += read;
-+
-+ if (err) {
-+ cxgb3i_log_info("conn 0x%p rx failed err %d.\n", conn, err);
-+ iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
-+ }
-+}
-+
-+void cxgb3i_conn_tx_open(struct s3_conn *c3cn)
-+{
-+ struct iscsi_conn *conn = c3cn->user_data;
-+ struct iscsi_tcp_conn *tcp_conn;
-+
-+ cxgb3i_tx_debug("cn 0x%p.\n", c3cn);
-+ if (conn) {
-+ cxgb3i_tx_debug("cn 0x%p, cid %d.\n", c3cn, conn->id);
-+ tcp_conn = conn->dd_data;
-+ scsi_queue_work(conn->session->host, &conn->xmitwork);
-+ }
-+}
-+
-+void cxgb3i_conn_closing(struct s3_conn *c3cn)
-+{
-+ struct iscsi_conn *conn;
-+
-+ read_lock(&c3cn->callback_lock);
-+ conn = c3cn->user_data;
-+ if (conn && c3cn->state != C3CN_STATE_ESTABLISHED)
-+ iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
-+ read_unlock(&c3cn->callback_lock);
-+}
-+
-+int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
-+{
-+ struct t3cdev *tdev = snic->tdev;
-+ struct cxgb3i_ddp_info *ddp = &snic->ddp;
-+ struct ulp_iscsi_info uinfo;
-+ unsigned int ppmax, bits, max_bits;
-+ int i, err;
-+
-+ spin_lock_init(&ddp->map_lock);
-+
-+ err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
-+ if (err < 0) {
-+ cxgb3i_log_error("%s, failed to get iscsi param err=%d.\n",
-+ tdev->name, err);
-+ return err;
-+ }
-+
-+ ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
-+ max_bits = min(PPOD_IDX_MAX_SIZE,
-+ (32 - sw_tag_idx_bits - sw_tag_age_bits));
-+ bits = __ilog2_u32(ppmax) + 1;
-+ if (bits > max_bits)
-+ bits = max_bits;
-+ ppmax = (1 << bits) - 1;
-+
-+ snic->tx_max_size = min_t(unsigned int,
-+ uinfo.max_txsz, ULP2_MAX_PKT_SIZE);
-+ snic->rx_max_size = min_t(unsigned int,
-+ uinfo.max_rxsz, ULP2_MAX_PKT_SIZE);
-+
-+ snic->tag_format.idx_bits = sw_tag_idx_bits;
-+ snic->tag_format.age_bits = sw_tag_age_bits;
-+ snic->tag_format.rsvd_bits = bits;
-+ snic->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
-+ snic->tag_format.rsvd_mask = (1 << snic->tag_format.rsvd_bits) - 1;
-+ snic->tag_format.rsvd_tag_mask =
-+ (1 << (snic->tag_format.rsvd_bits + PPOD_IDX_SHIFT)) - 1;
-+
-+ ddp->map = cxgb3i_alloc_big_mem(ppmax);
-+ if (!ddp->map) {
-+ cxgb3i_log_warn("snic unable to alloc ddp ppod 0x%u, "
-+ "ddp disabled.\n", ppmax);
-+ return 0;
-+ }
-+ ddp->llimit = uinfo.llimit;
-+ ddp->ulimit = uinfo.ulimit;
-+
-+ uinfo.tagmask =
-+ snic->tag_format.rsvd_mask << snic->tag_format.rsvd_shift;
-+ for (i = 0; i < ULP2_PGIDX_MAX; i++)
-+ uinfo.pgsz_factor[i] = ddp_page_order[i];
-+
-+ uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
-+
-+ err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
-+ if (err < 0) {
-+ cxgb3i_log_warn("snic unable to set iscsi param err=%d, "
-+ "ddp disabled.\n", err);
-+ goto free_ppod_map;
-+ }
-+
-+ ddp->nppods = ppmax;
-+ ddp->idx_last = ppmax;
-+
-+ tdev->ulp_iscsi = ddp;
-+
-+ cxgb3i_log_info("snic nppods %u (0x%x ~ 0x%x), rsvd shift %u, "
-+ "bits %u, mask 0x%x, 0x%x, pkt %u,%u.\n",
-+ ppmax, ddp->llimit, ddp->ulimit,
-+ snic->tag_format.rsvd_shift,
-+ snic->tag_format.rsvd_bits,
-+ snic->tag_format.rsvd_mask, uinfo.tagmask,
-+ snic->tx_max_size, snic->rx_max_size);
-+
-+ return 0;
-+
-+free_ppod_map:
-+ cxgb3i_free_big_mem(ddp->map);
-+ return 0;
-+}
-+
-+void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *snic)
-+{
-+ u8 *map = snic->ddp.map;
-+
-+ if (map) {
-+ snic->tdev->ulp_iscsi = NULL;
-+ spin_lock(&snic->lock);
-+ snic->ddp.map = NULL;
-+ spin_unlock(&snic->lock);
-+ cxgb3i_free_big_mem(map);
-+ }
-+}
---- /dev/null
-+++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
-@@ -0,0 +1,108 @@
-+/*
-+ * cxgb3i_ulp2.h: Chelsio S3xx iSCSI driver.
-+ *
-+ * Copyright (c) 2008 Chelsio Communications, Inc.
-+ *
-+ * This program is free software; you can redistribute it and/or modify
-+ * it under the terms of the GNU General Public License as published by
-+ * the Free Software Foundation.
-+ *
-+ * Written by: Karen Xie (kxie@chelsio.com)
-+ */
-+
-+#ifndef __CXGB3I_ULP2_H__
-+#define __CXGB3I_ULP2_H__
-+
-+#define ULP2_PDU_PAYLOAD_DFLT (16224 - ISCSI_PDU_HEADER_MAX)
-+#define PPOD_PAGES_MAX 4
-+#define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */
-+
-+struct pagepod_hdr {
-+ u32 vld_tid;
-+ u32 pgsz_tag_clr;
-+ u32 maxoffset;
-+ u32 pgoffset;
-+ u64 rsvd;
-+};
-+
-+struct pagepod {
-+ struct pagepod_hdr hdr;
-+ u64 addr[PPOD_PAGES_MAX + 1];
-+};
-+
-+#define PPOD_SIZE sizeof(struct pagepod) /* 64 */
-+#define PPOD_SIZE_SHIFT 6
-+
-+#define PPOD_COLOR_SHIFT 0
-+#define PPOD_COLOR_SIZE 6
-+#define PPOD_COLOR_MASK ((1 << PPOD_COLOR_SIZE) - 1)
-+
-+#define PPOD_IDX_SHIFT PPOD_COLOR_SIZE
-+#define PPOD_IDX_MAX_SIZE 24
-+
-+#define S_PPOD_TID 0
-+#define M_PPOD_TID 0xFFFFFF
-+#define V_PPOD_TID(x) ((x) << S_PPOD_TID)
-+
-+#define S_PPOD_VALID 24
-+#define V_PPOD_VALID(x) ((x) << S_PPOD_VALID)
-+#define F_PPOD_VALID V_PPOD_VALID(1U)
-+
-+#define S_PPOD_COLOR 0
-+#define M_PPOD_COLOR 0x3F
-+#define V_PPOD_COLOR(x) ((x) << S_PPOD_COLOR)
-+
-+#define S_PPOD_TAG 6
-+#define M_PPOD_TAG 0xFFFFFF
-+#define V_PPOD_TAG(x) ((x) << S_PPOD_TAG)
-+
-+#define S_PPOD_PGSZ 30
-+#define M_PPOD_PGSZ 0x3
-+#define V_PPOD_PGSZ(x) ((x) << S_PPOD_PGSZ)
-+
-+struct cpl_iscsi_hdr_norss {
-+ union opcode_tid ot;
-+ u16 pdu_len_ddp;
-+ u16 len;
-+ u32 seq;
-+ u16 urg;
-+ u8 rsvd;
-+ u8 status;
-+};
-+
-+struct cpl_rx_data_ddp_norss {
-+ union opcode_tid ot;
-+ u16 urg;
-+ u16 len;
-+ u32 seq;
-+ u32 nxt_seq;
-+ u32 ulp_crc;
-+ u32 ddp_status;
-+};
-+
-+#define RX_DDP_STATUS_IPP_SHIFT 27 /* invalid pagepod */
-+#define RX_DDP_STATUS_TID_SHIFT 26 /* tid mismatch */
-+#define RX_DDP_STATUS_COLOR_SHIFT 25 /* color mismatch */
-+#define RX_DDP_STATUS_OFFSET_SHIFT 24 /* offset mismatch */
-+#define RX_DDP_STATUS_ULIMIT_SHIFT 23 /* ulimit error */
-+#define RX_DDP_STATUS_TAG_SHIFT 22 /* tag mismatch */
-+#define RX_DDP_STATUS_DCRC_SHIFT 21 /* dcrc error */
-+#define RX_DDP_STATUS_HCRC_SHIFT 20 /* hcrc error */
-+#define RX_DDP_STATUS_PAD_SHIFT 19 /* pad error */
-+#define RX_DDP_STATUS_PPP_SHIFT 18 /* pagepod parity error */
-+#define RX_DDP_STATUS_LLIMIT_SHIFT 17 /* llimit error */
-+#define RX_DDP_STATUS_DDP_SHIFT 16 /* ddp'able */
-+#define RX_DDP_STATUS_PMM_SHIFT 15 /* pagepod mismatch */
-+
-+#define ULP2_FLAG_DATA_READY 0x1
-+#define ULP2_FLAG_DATA_DDPED 0x2
-+#define ULP2_FLAG_HCRC_ERROR 0x10
-+#define ULP2_FLAG_DCRC_ERROR 0x20
-+#define ULP2_FLAG_PAD_ERROR 0x40
-+
-+#define ULP2_MAX_PKT_SIZE 16224
-+
-+void cxgb3i_conn_closing(struct s3_conn *);
-+void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn);
-+void cxgb3i_conn_tx_open(struct s3_conn *c3cn);
-+#endif
---- /dev/null
-+++ b/drivers/scsi/cxgb3i/Kconfig
-@@ -0,0 +1,7 @@
-+config SCSI_CXGB3_ISCSI
-+ tristate "Chelsio S3xx iSCSI support"
-+ select CHELSIO_T3
-+ select SCSI_ISCSI_ATTRS
-+ select ISCSI_TCP
-+ ---help---
-+ This driver supports iSCSI offload for the Chelsio S3 series devices.
---- /dev/null
-+++ b/drivers/scsi/cxgb3i/Makefile
-@@ -0,0 +1,5 @@
-+EXTRA_CFLAGS += -I$(TOPDIR)/drivers/net/cxgb3
-+
-+cxgb3i-y := cxgb3i_init.o cxgb3i_iscsi.o cxgb3i_ulp2.o cxgb3i_offload.o
-+
-+obj-$(CONFIG_SCSI_CXGB3_ISCSI) += cxgb3i.o
---- a/drivers/scsi/Kconfig
-+++ b/drivers/scsi/Kconfig
-@@ -352,6 +352,8 @@ config ISCSI_TCP
-
- http://open-iscsi.org
-
-+source "drivers/scsi/cxgb3i/Kconfig"
-+
- config SGIWD93_SCSI
- tristate "SGI WD93C93 SCSI Driver"
- depends on SGI_HAS_WD93 && SCSI
---- a/drivers/scsi/Makefile
-+++ b/drivers/scsi/Makefile
-@@ -40,6 +40,7 @@ obj-$(CONFIG_LIBFC) += libfc/
- obj-$(CONFIG_FCOE) += fcoe/
- obj-$(CONFIG_ISCSI_TCP) += libiscsi.o iscsi_tcp.o
- obj-$(CONFIG_INFINIBAND_ISER) += libiscsi.o
-+obj-$(CONFIG_SCSI_CXGB3_ISCSI) += iscsi_tcp.o cxgb3i/
- obj-$(CONFIG_SCSI_A4000T) += 53c700.o a4000t.o
- obj-$(CONFIG_SCSI_ZORRO7XX) += 53c700.o zorro7xx.o
- obj-$(CONFIG_A3000_SCSI) += a3000.o wd33c93.o
projects / ipfire-2.x.git / blobdiff