--- /dev/null
+Subject: add cxgb3i iscsi driver
+From: Karen Xie <kxie@chelsio.com>
+References: FATE#304154,bnc#433500
+
+Add Chelsio S3 iscsi initiator driver.
+This patch implements the cxgb3i iscsi connection offload.
+
+Signed-off-by: Karen Xie <kxie@chelsio.com>
+Signed-off-by: Chandra Seetharaman <sekharan@us.ibm.com>
+Signed-off-by: Hannes Reinecke <hare@suse.de>
+---
+
+---
+ drivers/scsi/Kconfig | 2
+ drivers/scsi/Makefile | 1
+ drivers/scsi/cxgb3i/Kconfig | 7
+ drivers/scsi/cxgb3i/Makefile | 5
+ drivers/scsi/cxgb3i/cxgb3i.h | 179 +++
+ drivers/scsi/cxgb3i/cxgb3i_init.c | 109 +
+ drivers/scsi/cxgb3i/cxgb3i_iscsi.c | 854 ++++++++++++++
+ drivers/scsi/cxgb3i/cxgb3i_offload.c | 2021 +++++++++++++++++++++++++++++++++++
+ drivers/scsi/cxgb3i/cxgb3i_offload.h | 220 +++
+ drivers/scsi/cxgb3i/cxgb3i_ulp2.c | 741 ++++++++++++
+ drivers/scsi/cxgb3i/cxgb3i_ulp2.h | 108 +
+ 11 files changed, 4247 insertions(+)
+
+--- /dev/null
++++ b/drivers/scsi/cxgb3i/cxgb3i.h
+@@ -0,0 +1,179 @@
++/*
++ * cxgb3i.h: Chelsio S3xx iSCSI driver.
++ *
++ * Copyright (c) 2008 Chelsio Communications, Inc.
++ *
++ * This program is free software; you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License as published by
++ * the Free Software Foundation.
++ *
++ * Written by: Karen Xie (kxie@chelsio.com)
++ */
++
++#ifndef __CXGB3I_H__
++#define __CXGB3I_H__
++
++#include <linux/module.h>
++#include <linux/moduleparam.h>
++#include <linux/errno.h>
++#include <linux/types.h>
++#include <linux/list.h>
++#include <linux/netdevice.h>
++#include <linux/scatterlist.h>
++
++/* from cxgb3 LLD */
++#include "common.h"
++#include "t3_cpl.h"
++#include "t3cdev.h"
++#include "cxgb3_ctl_defs.h"
++#include "cxgb3_offload.h"
++#include "firmware_exports.h"
++#include "cxgb3i_offload.h"
++/* from iscsi */
++#include "../iscsi_tcp.h"
++
++#define CXGB3I_SCSI_QDEPTH_DFLT 128
++#define CXGB3I_MAX_TARGET CXGB3I_MAX_CONN
++#define CXGB3I_MAX_LUN 512
++#define ISCSI_PDU_HEADER_MAX (56 + 256) /* bhs + digests + ahs */
++
++struct cxgb3i_adapter;
++struct cxgb3i_hba;
++struct cxgb3i_endpoint;
++
++/**
++ * struct cxgb3i_tag_format - cxgb3i ulp tag for steering pdu payload
++ *
++ * @idx_bits: # of bits used to store itt (from iscsi laryer)
++ * @age_bits: # of bits used to store age (from iscsi laryer)
++ * @rsvd_bits: # of bits used by h/w
++ * @rsvd_shift: shift left
++ * @rsvd_mask: bit mask
++ * @rsvd_tag_mask: h/w tag bit mask
++ *
++ */
++struct cxgb3i_tag_format {
++ unsigned char idx_bits;
++ unsigned char age_bits;
++ unsigned char rsvd_bits;
++ unsigned char rsvd_shift;
++ u32 rsvd_mask;
++ u32 rsvd_tag_mask;
++};
++
++/**
++ * struct cxgb3i_ddp_info - cxgb3i direct data placement for pdu payload
++ *
++ * @llimit: lower bound of the page pod memory
++ * @ulimit: upper bound of the page pod memory
++ * @nppods: # of page pod entries
++ * @idx_last: page pod entry last used
++ * @map_lock: lock to synchonize access to the page pod map
++ * @map: page pod map
++ */
++struct cxgb3i_ddp_info {
++ unsigned int llimit;
++ unsigned int ulimit;
++ unsigned int nppods;
++ unsigned int idx_last;
++ spinlock_t map_lock;
++ u8 *map;
++};
++
++/**
++ * struct cxgb3i_hba - cxgb3i iscsi structure (per port)
++ *
++ * @snic: cxgb3i adapter containing this port
++ * @ndev: pointer to netdev structure
++ * @shost: pointer to scsi host structure
++ */
++struct cxgb3i_hba {
++ struct cxgb3i_adapter *snic;
++ struct net_device *ndev;
++ struct Scsi_Host *shost;
++};
++
++/**
++ * struct cxgb3i_adapter - cxgb3i adapter structure (per pci)
++ *
++ * @listhead: list head to link elements
++ * @lock: lock for this structure
++ * @tdev: pointer to t3cdev used by cxgb3 driver
++ * @pdev: pointer to pci dev
++ * @hba_cnt: # of hbas (the same as # of ports)
++ * @hba: all the hbas on this adapter
++ * @tx_max_size: max. tx packet size supported
++ * @rx_max_size: max. rx packet size supported
++ * @tag_format: ulp tag format settings
++ * @ddp: ulp ddp state
++ */
++struct cxgb3i_adapter {
++ struct list_head list_head;
++ spinlock_t lock;
++ struct t3cdev *tdev;
++ struct pci_dev *pdev;
++ unsigned char hba_cnt;
++ struct cxgb3i_hba *hba[MAX_NPORTS];
++
++ unsigned int tx_max_size;
++ unsigned int rx_max_size;
++
++ struct cxgb3i_tag_format tag_format;
++ struct cxgb3i_ddp_info ddp;
++};
++
++/**
++ * struct cxgb3i_conn - cxgb3i iscsi connection
++ *
++ * @tcp_conn: pointer to iscsi_tcp_conn structure
++ * @listhead: list head to link elements
++ * @conn: pointer to iscsi_conn structure
++ * @hba: pointer to the hba this conn. is going through
++ */
++struct cxgb3i_conn {
++ struct iscsi_tcp_conn tcp_conn;
++ struct list_head list_head;
++ struct cxgb3i_endpoint *cep;
++ struct iscsi_conn *conn;
++ struct cxgb3i_hba *hba;
++};
++
++/**
++ * struct cxgb3i_endpoint - iscsi tcp endpoint
++ *
++ * @c3cn: the h/w tcp connection representation
++ * @hba: pointer to the hba this conn. is going through
++ * @cconn: pointer to the associated cxgb3i iscsi connection
++ */
++struct cxgb3i_endpoint {
++ struct s3_conn *c3cn;
++ struct cxgb3i_hba *hba;
++ struct cxgb3i_conn *cconn;
++};
++
++/*
++ * Function Prototypes
++ */
++int cxgb3i_iscsi_init(void);
++void cxgb3i_iscsi_cleanup(void);
++
++struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *);
++void cxgb3i_adapter_remove(struct t3cdev *);
++int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *);
++void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *);
++
++struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *);
++struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *,
++ struct net_device *);
++void cxgb3i_hba_host_remove(struct cxgb3i_hba *);
++
++int cxgb3i_ulp2_init(void);
++void cxgb3i_ulp2_cleanup(void);
++int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *, int, int);
++void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32,
++ struct scatterlist *, unsigned int);
++u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *, unsigned int,
++ u32, unsigned int, struct scatterlist *,
++ unsigned int);
++int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *);
++#endif
+--- /dev/null
++++ b/drivers/scsi/cxgb3i/cxgb3i_init.c
+@@ -0,0 +1,109 @@
++/* cxgb3i_init.c: Chelsio S3xx iSCSI driver.
++ *
++ * Copyright (c) 2008 Chelsio Communications, Inc.
++ *
++ * This program is free software; you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License as published by
++ * the Free Software Foundation.
++ *
++ * Written by: Karen Xie (kxie@chelsio.com)
++ */
++
++#include "cxgb3i.h"
++
++#define DRV_MODULE_NAME "cxgb3i"
++#define DRV_MODULE_VERSION "1.0.0"
++#define DRV_MODULE_RELDATE "Jun. 1, 2008"
++
++static char version[] =
++ "Chelsio S3xx iSCSI Driver " DRV_MODULE_NAME
++ " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
++
++MODULE_AUTHOR("Karen Xie <kxie@chelsio.com>");
++MODULE_DESCRIPTION("Chelsio S3xx iSCSI Driver");
++MODULE_LICENSE("GPL");
++MODULE_VERSION(DRV_MODULE_VERSION);
++
++static void open_s3_dev(struct t3cdev *);
++static void close_s3_dev(struct t3cdev *);
++
++static cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS];
++static struct cxgb3_client t3c_client = {
++ .name = "iscsi_cxgb3",
++ .handlers = cxgb3i_cpl_handlers,
++ .add = open_s3_dev,
++ .remove = close_s3_dev,
++};
++
++/**
++ * open_s3_dev - register with cxgb3 LLD
++ * @t3dev: cxgb3 adapter instance
++ */
++static void open_s3_dev(struct t3cdev *t3dev)
++{
++ static int vers_printed;
++
++ if (!vers_printed) {
++ printk(KERN_INFO "%s", version);
++ vers_printed = 1;
++ }
++
++ cxgb3i_log_debug("open cxgb3 %s.\n", t3dev->name);
++ cxgb3i_sdev_add(t3dev, &t3c_client);
++ cxgb3i_adapter_add(t3dev);
++}
++
++/**
++ * close_s3_dev - de-register with cxgb3 LLD
++ * @t3dev: cxgb3 adapter instance
++ */
++static void close_s3_dev(struct t3cdev *t3dev)
++{
++ cxgb3i_log_debug("close cxgb3 %s.\n", t3dev->name);
++ cxgb3i_adapter_remove(t3dev);
++ cxgb3i_sdev_remove(t3dev);
++}
++
++/**
++ * cxgb3i_init_module - module init entry point
++ *
++ * initialize any driver wide global data structures and register itself
++ * with the cxgb3 module
++ */
++static int __init cxgb3i_init_module(void)
++{
++ int err;
++
++ err = cxgb3i_sdev_init(cxgb3i_cpl_handlers);
++ if (err < 0)
++ return err;
++
++ err = cxgb3i_iscsi_init();
++ if (err < 0)
++ return err;
++
++ err = cxgb3i_ulp2_init();
++ if (err < 0)
++ return err;
++
++ cxgb3_register_client(&t3c_client);
++
++ return 0;
++}
++
++/**
++ * cxgb3i_exit_module - module cleanup/exit entry point
++ *
++ * go through the driver hba list and for each hba, release any resource held.
++ * and unregisters iscsi transport and the cxgb3 module
++ */
++static void __exit cxgb3i_exit_module(void)
++{
++ cxgb3_unregister_client(&t3c_client);
++ cxgb3i_ulp2_cleanup();
++ cxgb3i_iscsi_cleanup();
++ cxgb3i_sdev_cleanup();
++}
++
++module_init(cxgb3i_init_module);
++module_exit(cxgb3i_exit_module);
+--- /dev/null
++++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
+@@ -0,0 +1,854 @@
++/* cxgb3i_iscsi.c: Chelsio S3xx iSCSI driver.
++ *
++ * Copyright (c) 2008 Chelsio Communications, Inc.
++ *
++ * This program is free software; you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License as published by
++ * the Free Software Foundation.
++ *
++ * Written by: Karen Xie (kxie@chelsio.com)
++ */
++
++#include <linux/inet.h>
++#include <linux/crypto.h>
++#include <net/tcp.h>
++#include <scsi/scsi_cmnd.h>
++#include <scsi/scsi_device.h>
++#include <scsi/scsi_eh.h>
++#include <scsi/scsi_host.h>
++#include <scsi/scsi.h>
++#include <scsi/iscsi_proto.h>
++#include <scsi/libiscsi.h>
++#include <scsi/scsi_transport_iscsi.h>
++
++#include "cxgb3i.h"
++
++static struct scsi_transport_template *cxgb3i_scsi_transport;
++static struct scsi_host_template cxgb3i_host_template;
++static struct iscsi_transport cxgb3i_iscsi_transport;
++
++static LIST_HEAD(cxgb3i_snic_list);
++static DEFINE_RWLOCK(cxgb3i_snic_rwlock);
++
++/**
++ * cxgb3i_adapter_add - init a s3 adapter structure and any h/w settings
++ * @snic: pointer to adapter instance
++ */
++struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *t3dev)
++{
++ struct cxgb3i_adapter *snic;
++ struct adapter *adapter = tdev2adap(t3dev);
++ int i;
++
++ snic = kzalloc(sizeof(*snic), GFP_KERNEL);
++ if (!snic) {
++ cxgb3i_log_debug("cxgb3 %s, OOM.\n", t3dev->name);
++ return NULL;
++ }
++
++ spin_lock_init(&snic->lock);
++ snic->tdev = t3dev;
++ snic->pdev = adapter->pdev;
++
++ if (cxgb3i_adapter_ulp_init(snic))
++ goto free_snic;
++
++ for_each_port(adapter, i) {
++ snic->hba[i] = cxgb3i_hba_host_add(snic, adapter->port[i]);
++ if (!snic->hba[i])
++ goto ulp_cleanup;
++ }
++ snic->hba_cnt = adapter->params.nports;
++
++ /* add to the list */
++ write_lock(&cxgb3i_snic_rwlock);
++ list_add_tail(&snic->list_head, &cxgb3i_snic_list);
++ write_unlock(&cxgb3i_snic_rwlock);
++
++ return snic;
++
++ulp_cleanup:
++ cxgb3i_adapter_ulp_cleanup(snic);
++free_snic:
++ kfree(snic);
++ return NULL;
++}
++
++/**
++ * cxgb3i_snic_cleanup - release all the resources held and cleanup h/w settings
++ * @snic: pointer to adapter instance
++ */
++void cxgb3i_adapter_remove(struct t3cdev *t3dev)
++{
++ int i;
++ struct cxgb3i_adapter *snic;
++
++ /* remove from the list */
++ read_lock(&cxgb3i_snic_rwlock);
++ list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
++ if (snic->tdev == t3dev) {
++ list_del(&snic->list_head);
++ break;
++ }
++ }
++ write_unlock(&cxgb3i_snic_rwlock);
++
++ if (snic) {
++ for (i = 0; i < snic->hba_cnt; i++) {
++ if (snic->hba[i]) {
++ cxgb3i_hba_host_remove(snic->hba[i]);
++ snic->hba[i] = NULL;
++ }
++ }
++
++ /* release ddp resources */
++ cxgb3i_adapter_ulp_cleanup(snic);
++ kfree(snic);
++ }
++}
++
++struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *ndev)
++{
++ struct cxgb3i_adapter *snic;
++ int i;
++
++ read_lock(&cxgb3i_snic_rwlock);
++ list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
++ for (i = 0; i < snic->hba_cnt; i++) {
++ if (snic->hba[i]->ndev == ndev) {
++ read_unlock(&cxgb3i_snic_rwlock);
++ return snic->hba[i];
++ }
++ }
++ }
++ read_unlock(&cxgb3i_snic_rwlock);
++ return NULL;
++}
++
++struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *snic,
++ struct net_device *ndev)
++{
++ struct cxgb3i_hba *hba;
++ struct Scsi_Host *shost;
++ int err;
++
++ shost = iscsi_host_alloc(&cxgb3i_host_template,
++ sizeof(struct cxgb3i_hba),
++ CXGB3I_SCSI_QDEPTH_DFLT);
++ if (!shost) {
++ cxgb3i_log_info("iscsi_host_alloc failed.\n");
++ return NULL;
++ }
++
++ shost->transportt = cxgb3i_scsi_transport;
++ shost->max_lun = CXGB3I_MAX_LUN;
++ shost->max_id = CXGB3I_MAX_TARGET;
++ shost->max_channel = 0;
++ shost->max_cmd_len = 16;
++
++ hba = iscsi_host_priv(shost);
++ hba->snic = snic;
++ hba->ndev = ndev;
++ hba->shost = shost;
++
++ pci_dev_get(snic->pdev);
++ err = iscsi_host_add(shost, &snic->pdev->dev);
++ if (err) {
++ cxgb3i_log_info("iscsi_host_add failed.\n");
++ goto pci_dev_put;
++ }
++
++ cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n",
++ shost, hba, shost->host_no);
++
++ return hba;
++
++pci_dev_put:
++ pci_dev_put(snic->pdev);
++ scsi_host_put(shost);
++ return NULL;
++}
++
++void cxgb3i_hba_host_remove(struct cxgb3i_hba *hba)
++{
++ cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n",
++ hba->shost, hba, hba->shost->host_no);
++ iscsi_host_remove(hba->shost);
++ pci_dev_put(hba->snic->pdev);
++ iscsi_host_free(hba->shost);
++}
++
++/**
++ * cxgb3i_ep_connect - establish TCP connection to target portal
++ * @dst_addr: target IP address
++ * @non_blocking: blocking or non-blocking call
++ *
++ * Initiates a TCP/IP connection to the dst_addr
++ */
++static struct iscsi_endpoint *cxgb3i_ep_connect(struct sockaddr *dst_addr,
++ int non_blocking)
++{
++ struct iscsi_endpoint *ep;
++ struct cxgb3i_endpoint *cep;
++ struct cxgb3i_hba *hba;
++ struct s3_conn *c3cn = NULL;
++ int err = 0;
++
++ c3cn = cxgb3i_c3cn_create();
++ if (!c3cn) {
++ cxgb3i_log_info("ep connect OOM.\n");
++ err = -ENOMEM;
++ goto release_conn;
++ }
++
++ err = cxgb3i_c3cn_connect(c3cn, (struct sockaddr_in *)dst_addr);
++ if (err < 0) {
++ cxgb3i_log_info("ep connect failed.\n");
++ goto release_conn;
++ }
++ hba = cxgb3i_hba_find_by_netdev(c3cn->dst_cache->dev);
++ if (!hba) {
++ err = -ENOSPC;
++ cxgb3i_log_info("NOT going through cxgbi device.\n");
++ goto release_conn;
++ }
++ if (c3cn_in_state(c3cn, C3CN_STATE_CLOSE)) {
++ err = -ENOSPC;
++ cxgb3i_log_info("ep connect unable to connect.\n");
++ goto release_conn;
++ }
++
++ ep = iscsi_create_endpoint(sizeof(*cep));
++ if (!ep) {
++ err = -ENOMEM;
++ cxgb3i_log_info("iscsi alloc ep, OOM.\n");
++ goto release_conn;
++ }
++ cep = ep->dd_data;
++ cep->c3cn = c3cn;
++ cep->hba = hba;
++
++ cxgb3i_log_debug("ep 0x%p, 0x%p, c3cn 0x%p, hba 0x%p.\n",
++ ep, cep, c3cn, hba);
++ return ep;
++
++release_conn:
++ cxgb3i_log_debug("conn 0x%p failed, release.\n", c3cn);
++ if (c3cn)
++ cxgb3i_c3cn_release(c3cn);
++ return ERR_PTR(err);
++}
++
++/**
++ * cxgb3i_ep_poll - polls for TCP connection establishement
++ * @ep: TCP connection (endpoint) handle
++ * @timeout_ms: timeout value in milli secs
++ *
++ * polls for TCP connect request to complete
++ */
++static int cxgb3i_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
++{
++ struct cxgb3i_endpoint *cep = ep->dd_data;
++ struct s3_conn *c3cn = cep->c3cn;
++
++ if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED))
++ return 0;
++ cxgb3i_log_debug("ep 0x%p, c3cn 0x%p established.\n", ep, c3cn);
++ return 1;
++}
++
++/**
++ * cxgb3i_ep_disconnect - teardown TCP connection
++ * @ep: TCP connection (endpoint) handle
++ *
++ * teardown TCP connection
++ */
++static void cxgb3i_ep_disconnect(struct iscsi_endpoint *ep)
++{
++ struct cxgb3i_endpoint *cep = ep->dd_data;
++ struct cxgb3i_conn *cconn = cep->cconn;
++
++ cxgb3i_log_debug("ep 0x%p, cep 0x%p.\n", ep, cep);
++
++ if (cconn && cconn->conn) {
++ struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
++
++ /*
++ * stop the xmit path so the xmit_segment function is
++ * not being called
++ */
++ write_lock_bh(&cep->c3cn->callback_lock);
++ set_bit(ISCSI_SUSPEND_BIT, &cconn->conn->suspend_rx);
++ cep->c3cn->user_data = NULL;
++ cconn->cep = NULL;
++ tcp_conn->sock = NULL;
++ write_unlock_bh(&cep->c3cn->callback_lock);
++ }
++
++ cxgb3i_log_debug("ep 0x%p, cep 0x%p, release c3cn 0x%p.\n",
++ ep, cep, cep->c3cn);
++ cxgb3i_c3cn_release(cep->c3cn);
++ iscsi_destroy_endpoint(ep);
++}
++
++/**
++ * cxgb3i_session_create - create a new iscsi session
++ * @cmds_max: max # of commands
++ * @qdepth: scsi queue depth
++ * @initial_cmdsn: initial iscsi CMDSN for this session
++ * @host_no: pointer to return host no
++ *
++ * Creates a new iSCSI session
++ */
++static struct iscsi_cls_session *
++cxgb3i_session_create(struct iscsi_endpoint *ep, u16 cmds_max, u16 qdepth,
++ u32 initial_cmdsn, u32 *host_no)
++{
++ struct cxgb3i_endpoint *cep;
++ struct cxgb3i_hba *hba;
++ struct Scsi_Host *shost;
++ struct iscsi_cls_session *cls_session;
++ struct iscsi_session *session;
++ int i;
++
++ if (!ep) {
++ cxgb3i_log_error("%s, missing endpoint.\n", __func__);
++ return NULL;
++ }
++
++ cep = ep->dd_data;
++ hba = cep->hba;
++ shost = hba->shost;
++ cxgb3i_log_debug("ep 0x%p, cep 0x%p, hba 0x%p.\n", ep, cep, hba);
++ BUG_ON(hba != iscsi_host_priv(shost));
++
++ *host_no = shost->host_no;
++
++ cls_session = iscsi_session_setup(&cxgb3i_iscsi_transport, shost,
++ cmds_max,
++ sizeof(struct iscsi_tcp_task),
++ initial_cmdsn, ISCSI_MAX_TARGET);
++ if (!cls_session)
++ return NULL;
++ session = cls_session->dd_data;
++
++ for (i = 0; i < session->cmds_max; i++) {
++ struct iscsi_task *task = session->cmds[i];
++ struct iscsi_tcp_task *tcp_task = task->dd_data;
++
++ task->hdr = &tcp_task->hdr.cmd_hdr;
++ task->hdr_max = sizeof(tcp_task->hdr) - ISCSI_DIGEST_SIZE;
++ }
++
++ if (iscsi_r2tpool_alloc(session))
++ goto remove_session;
++
++ return cls_session;
++
++remove_session:
++ iscsi_session_teardown(cls_session);
++ return NULL;
++}
++
++/**
++ * cxgb3i_session_destroy - destroys iscsi session
++ * @cls_session: pointer to iscsi cls session
++ *
++ * Destroys an iSCSI session instance and releases its all resources held
++ */
++static void cxgb3i_session_destroy(struct iscsi_cls_session *cls_session)
++{
++ cxgb3i_log_debug("sess 0x%p.\n", cls_session);
++ iscsi_r2tpool_free(cls_session->dd_data);
++ iscsi_session_teardown(cls_session);
++}
++
++/**
++ * cxgb3i_conn_create - create iscsi connection instance
++ * @cls_session: pointer to iscsi cls session
++ * @cid: iscsi cid
++ *
++ * Creates a new iSCSI connection instance for a given session
++ */
++static inline void cxgb3i_conn_max_xmit_dlength(struct iscsi_conn *conn)
++{
++ struct cxgb3i_conn *cconn = conn->dd_data;
++
++ if (conn->max_xmit_dlength)
++ conn->max_xmit_dlength = min_t(unsigned int,
++ conn->max_xmit_dlength,
++ cconn->hba->snic->tx_max_size -
++ ISCSI_PDU_HEADER_MAX);
++ else
++ conn->max_xmit_dlength = cconn->hba->snic->tx_max_size -
++ ISCSI_PDU_HEADER_MAX;
++ cxgb3i_log_debug("conn 0x%p, max xmit %u.\n",
++ conn, conn->max_xmit_dlength);
++}
++
++static inline void cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
++{
++ struct cxgb3i_conn *cconn = conn->dd_data;
++
++ if (conn->max_recv_dlength)
++ conn->max_recv_dlength = min_t(unsigned int,
++ conn->max_recv_dlength,
++ cconn->hba->snic->rx_max_size -
++ ISCSI_PDU_HEADER_MAX);
++ else
++ conn->max_recv_dlength = cconn->hba->snic->rx_max_size -
++ ISCSI_PDU_HEADER_MAX;
++ cxgb3i_log_debug("conn 0x%p, max recv %u.\n",
++ conn, conn->max_recv_dlength);
++}
++
++static struct iscsi_cls_conn *cxgb3i_conn_create(struct iscsi_cls_session
++ *cls_session, u32 cid)
++{
++ struct iscsi_cls_conn *cls_conn;
++ struct iscsi_conn *conn;
++ struct cxgb3i_conn *cconn;
++
++ cxgb3i_log_debug("sess 0x%p, cid %u.\n", cls_session, cid);
++
++ cls_conn = iscsi_conn_setup(cls_session, sizeof(*cconn), cid);
++ if (!cls_conn)
++ return NULL;
++ conn = cls_conn->dd_data;
++
++ cconn = conn->dd_data;
++ cconn->tcp_conn.iscsi_conn = conn;
++ cconn->conn = conn;
++
++ return cls_conn;
++}
++
++/**
++ * cxgb3i_conn_xmit_segment - transmit segment
++ * @conn: pointer to iscsi conn
++ */
++static int cxgb3i_conn_xmit_segment(struct iscsi_conn *conn)
++{
++ struct cxgb3i_conn *cconn = conn->dd_data;
++ struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
++ struct iscsi_segment *segment = &tcp_conn->out.segment;
++
++ if (segment->total_copied < segment->total_size)
++ return cxgb3i_conn_ulp2_xmit(conn);
++ return 0;
++}
++
++/**
++ * cxgb3i_conn_bind - binds iscsi sess, conn and endpoint together
++ * @cls_session: pointer to iscsi cls session
++ * @cls_conn: pointer to iscsi cls conn
++ * @transport_eph: 64-bit EP handle
++ * @is_leading: leading connection on this session?
++ *
++ * Binds together an iSCSI session, an iSCSI connection and a
++ * TCP connection. This routine returns error code if the TCP
++ * connection does not belong on the device iSCSI sess/conn is bound
++ */
++
++static int cxgb3i_conn_bind(struct iscsi_cls_session *cls_session,
++ struct iscsi_cls_conn *cls_conn,
++ u64 transport_eph, int is_leading)
++{
++ struct iscsi_conn *conn = cls_conn->dd_data;
++ struct cxgb3i_conn *cconn = conn->dd_data;
++ struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
++ struct iscsi_endpoint *ep;
++ struct cxgb3i_endpoint *cep;
++ struct s3_conn *c3cn;
++ int err;
++
++ ep = iscsi_lookup_endpoint(transport_eph);
++ if (!ep)
++ return -EINVAL;
++
++ cxgb3i_log_debug("ep 0x%p, cls sess 0x%p, cls conn 0x%p.\n",
++ ep, cls_session, cls_conn);
++
++ err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
++ if (err)
++ return -EINVAL;
++
++ cep = ep->dd_data;
++ c3cn = cep->c3cn;
++
++ read_lock(&c3cn->callback_lock);
++ /* mnc: TODO don't abuse iscsi_tcp fields */
++ tcp_conn->sock = (struct socket *)c3cn;
++ c3cn->user_data = conn;
++ read_unlock(&c3cn->callback_lock);
++
++ cconn->hba = cep->hba;
++ cconn->cep = cep;
++ cep->cconn = cconn;
++
++ cxgb3i_conn_max_xmit_dlength(conn);
++ cxgb3i_conn_max_recv_dlength(conn);
++
++ spin_lock_bh(&conn->session->lock);
++ sprintf(conn->portal_address, NIPQUAD_FMT,
++ NIPQUAD(c3cn->daddr.sin_addr.s_addr));
++ conn->portal_port = ntohs(c3cn->daddr.sin_port);
++ spin_unlock_bh(&conn->session->lock);
++
++ tcp_conn->xmit_segment = cxgb3i_conn_xmit_segment;
++ iscsi_tcp_hdr_recv_prep(tcp_conn);
++
++ return 0;
++}
++
++/**
++ * cxgb3i_conn_get_param - return iscsi connection parameter to caller
++ * @cls_conn: pointer to iscsi cls conn
++ * @param: parameter type identifier
++ * @buf: buffer pointer
++ *
++ * returns iSCSI connection parameters
++ */
++static int cxgb3i_conn_get_param(struct iscsi_cls_conn *cls_conn,
++ enum iscsi_param param, char *buf)
++{
++ struct iscsi_conn *conn = cls_conn->dd_data;
++ int len;
++
++ cxgb3i_log_debug("cls_conn 0x%p, param %d.\n", cls_conn, param);
++
++ switch (param) {
++ case ISCSI_PARAM_CONN_PORT:
++ spin_lock_bh(&conn->session->lock);
++ len = sprintf(buf, "%hu\n", conn->portal_port);
++ spin_unlock_bh(&conn->session->lock);
++ break;
++ case ISCSI_PARAM_CONN_ADDRESS:
++ spin_lock_bh(&conn->session->lock);
++ len = sprintf(buf, "%s\n", conn->portal_address);
++ spin_unlock_bh(&conn->session->lock);
++ break;
++ default:
++ return iscsi_conn_get_param(cls_conn, param, buf);
++ }
++
++ return len;
++}
++
++static int cxgb3i_conn_set_param(struct iscsi_cls_conn *cls_conn,
++ enum iscsi_param param, char *buf, int buflen)
++{
++ struct iscsi_conn *conn = cls_conn->dd_data;
++ struct iscsi_session *session = conn->session;
++ struct cxgb3i_conn *cconn = conn->dd_data;
++ int value, err = 0;
++
++ switch (param) {
++ case ISCSI_PARAM_HDRDGST_EN:
++ err = iscsi_set_param(cls_conn, param, buf, buflen);
++ if (!err && conn->hdrdgst_en)
++ cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en,
++ conn->datadgst_en);
++ break;
++ case ISCSI_PARAM_DATADGST_EN:
++ err = iscsi_set_param(cls_conn, param, buf, buflen);
++ if (!err && conn->datadgst_en)
++ cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en,
++ conn->datadgst_en);
++ break;
++ case ISCSI_PARAM_MAX_R2T:
++ sscanf(buf, "%d", &value);
++ if (value <= 0 || !is_power_of_2(value))
++ return -EINVAL;
++ if (session->max_r2t == value)
++ break;
++ iscsi_r2tpool_free(session);
++ err = iscsi_set_param(cls_conn, param, buf, buflen);
++ if (!err && iscsi_r2tpool_alloc(session))
++ return -ENOMEM;
++ case ISCSI_PARAM_MAX_RECV_DLENGTH:
++ err = iscsi_set_param(cls_conn, param, buf, buflen);
++ cxgb3i_conn_max_recv_dlength(conn);
++ break;
++ case ISCSI_PARAM_MAX_XMIT_DLENGTH:
++ err = iscsi_set_param(cls_conn, param, buf, buflen);
++ cxgb3i_conn_max_xmit_dlength(conn);
++ break;
++ default:
++ return iscsi_set_param(cls_conn, param, buf, buflen);
++ }
++ return err;
++}
++
++/**
++ * cxgb3i_host_set_param - configure host (adapter) related parameters
++ * @shost: scsi host pointer
++ * @param: parameter type identifier
++ * @buf: buffer pointer
++ */
++static int cxgb3i_host_set_param(struct Scsi_Host *shost,
++ enum iscsi_host_param param,
++ char *buf, int buflen)
++{
++ struct cxgb3i_hba *hba = iscsi_host_priv(shost);
++
++ cxgb3i_log_debug("param %d, buf %s.\n", param, buf);
++
++ if (hba && param == ISCSI_HOST_PARAM_IPADDRESS) {
++ __be32 addr = in_aton(buf);
++ cxgb3i_set_private_ipv4addr(hba->ndev, addr);
++ return 0;
++ }
++
++ return iscsi_host_get_param(shost, param, buf);
++}
++
++/**
++ * cxgb3i_host_get_param - returns host (adapter) related parameters
++ * @shost: scsi host pointer
++ * @param: parameter type identifier
++ * @buf: buffer pointer
++ */
++static int cxgb3i_host_get_param(struct Scsi_Host *shost,
++ enum iscsi_host_param param, char *buf)
++{
++ struct cxgb3i_hba *hba = iscsi_host_priv(shost);
++ int i;
++ int len = 0;
++
++ cxgb3i_log_debug("hba %s, param %d.\n", hba->ndev->name, param);
++
++ switch (param) {
++ case ISCSI_HOST_PARAM_HWADDRESS:
++ for (i = 0; i < 6; i++)
++ len +=
++ sprintf(buf + len, "%02x.",
++ hba->ndev->dev_addr[i]);
++ len--;
++ buf[len] = '\0';
++ break;
++ case ISCSI_HOST_PARAM_NETDEV_NAME:
++ len = sprintf(buf, "%s\n", hba->ndev->name);
++ break;
++ case ISCSI_HOST_PARAM_IPADDRESS:
++ {
++ __be32 addr;
++
++ addr = cxgb3i_get_private_ipv4addr(hba->ndev);
++ len = sprintf(buf, "%u.%u.%u.%u", NIPQUAD(addr));
++ break;
++ }
++ default:
++ return iscsi_host_get_param(shost, param, buf);
++ }
++ return len;
++}
++
++/**
++ * cxgb3i_conn_get_stats - returns iSCSI stats
++ * @cls_conn: pointer to iscsi cls conn
++ * @stats: pointer to iscsi statistic struct
++ */
++static void cxgb3i_conn_get_stats(struct iscsi_cls_conn *cls_conn,
++ struct iscsi_stats *stats)
++{
++ struct iscsi_conn *conn = cls_conn->dd_data;
++
++ stats->txdata_octets = conn->txdata_octets;
++ stats->rxdata_octets = conn->rxdata_octets;
++ stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
++ stats->dataout_pdus = conn->dataout_pdus_cnt;
++ stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
++ stats->datain_pdus = conn->datain_pdus_cnt;
++ stats->r2t_pdus = conn->r2t_pdus_cnt;
++ stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
++ stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
++ stats->digest_err = 0;
++ stats->timeout_err = 0;
++ stats->custom_length = 1;
++ strcpy(stats->custom[0].desc, "eh_abort_cnt");
++ stats->custom[0].value = conn->eh_abort_cnt;
++}
++
++static inline u32 tag_base(struct cxgb3i_tag_format *format,
++ unsigned int idx, unsigned int age)
++{
++ u32 sw_bits = idx | (age << format->idx_bits);
++ u32 tag = sw_bits >> format->rsvd_shift;
++
++ tag <<= format->rsvd_bits + format->rsvd_shift;
++ tag |= sw_bits & ((1 << format->rsvd_shift) - 1);
++ return tag;
++}
++
++static inline void cxgb3i_parse_tag(struct cxgb3i_tag_format *format,
++ u32 tag, u32 *rsvd_bits, u32 *sw_bits)
++{
++ if (rsvd_bits)
++ *rsvd_bits = (tag >> format->rsvd_shift) & format->rsvd_mask;
++ if (sw_bits) {
++ *sw_bits = (tag >> (format->rsvd_shift + format->rsvd_bits))
++ << format->rsvd_shift;
++ *sw_bits |= tag & ((1 << format->rsvd_shift) - 1);
++ }
++}
++
++
++static void cxgb3i_parse_itt(struct iscsi_conn *conn, itt_t itt,
++ int *idx, int *age)
++{
++ struct cxgb3i_conn *cconn = conn->dd_data;
++ struct cxgb3i_adapter *snic = cconn->hba->snic;
++ u32 sw_bits;
++
++ cxgb3i_parse_tag(&snic->tag_format, itt, NULL, &sw_bits);
++ if (idx)
++ *idx = sw_bits & ISCSI_ITT_MASK;
++ if (age)
++ *age = (sw_bits >> snic->tag_format.idx_bits) & ISCSI_AGE_MASK;
++}
++
++static int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
++{
++ struct scsi_cmnd *sc = task->sc;
++ struct iscsi_conn *conn = task->conn;
++ struct iscsi_session *sess = conn->session;
++ struct cxgb3i_conn *cconn = conn->dd_data;
++ struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
++ struct cxgb3i_adapter *snic = cconn->hba->snic;
++ u32 sw_tag = tag_base(&snic->tag_format, task->itt, sess->age);
++ u32 tag = RESERVED_ITT;
++
++ if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE)) {
++ struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
++ tag =
++ cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag,
++ scsi_out(sc)->length,
++ scsi_out(sc)->table.sgl,
++ scsi_out(sc)->table.nents);
++ }
++ if (tag == RESERVED_ITT)
++ tag = sw_tag | (snic->tag_format.rsvd_mask <<
++ snic->tag_format.rsvd_shift);
++ *hdr_itt = htonl(tag);
++ return 0;
++}
++
++static void cxgb3i_release_itt(struct iscsi_task *task, itt_t hdr_itt)
++{
++ struct scsi_cmnd *sc = task->sc;
++ struct iscsi_conn *conn = task->conn;
++ struct cxgb3i_conn *cconn = conn->dd_data;
++ struct cxgb3i_adapter *snic = cconn->hba->snic;
++
++ hdr_itt = ntohl(hdr_itt);
++ if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE))
++ cxgb3i_ddp_tag_release(snic, hdr_itt,
++ scsi_out(sc)->table.sgl,
++ scsi_out(sc)->table.nents);
++}
++
++/**
++ * cxgb3i_host_template -- Scsi_Host_Template structure
++ * used when registering with the scsi mid layer
++ */
++static struct scsi_host_template cxgb3i_host_template = {
++ .module = THIS_MODULE,
++ .name = "Chelsio S3xx iSCSI Initiator",
++ .proc_name = "cxgb3i",
++ .queuecommand = iscsi_queuecommand,
++ .change_queue_depth = iscsi_change_queue_depth,
++ .can_queue = 128 * (ISCSI_DEF_XMIT_CMDS_MAX - 1),
++ .sg_tablesize = SG_ALL,
++ .max_sectors = 0xFFFF,
++ .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
++ .eh_abort_handler = iscsi_eh_abort,
++ .eh_device_reset_handler = iscsi_eh_device_reset,
++ .eh_target_reset_handler = iscsi_eh_host_reset,
++ .use_clustering = DISABLE_CLUSTERING,
++ .this_id = -1,
++};
++
++static struct iscsi_transport cxgb3i_iscsi_transport = {
++ .owner = THIS_MODULE,
++ .name = "cxgb3i",
++ .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
++ | CAP_DATADGST | CAP_DIGEST_OFFLOAD,
++ .param_mask = ISCSI_MAX_RECV_DLENGTH |
++ ISCSI_MAX_XMIT_DLENGTH |
++ ISCSI_HDRDGST_EN |
++ ISCSI_DATADGST_EN |
++ ISCSI_INITIAL_R2T_EN |
++ ISCSI_MAX_R2T |
++ ISCSI_IMM_DATA_EN |
++ ISCSI_FIRST_BURST |
++ ISCSI_MAX_BURST |
++ ISCSI_PDU_INORDER_EN |
++ ISCSI_DATASEQ_INORDER_EN |
++ ISCSI_ERL |
++ ISCSI_CONN_PORT |
++ ISCSI_CONN_ADDRESS |
++ ISCSI_EXP_STATSN |
++ ISCSI_PERSISTENT_PORT |
++ ISCSI_PERSISTENT_ADDRESS |
++ ISCSI_TARGET_NAME | ISCSI_TPGT |
++ ISCSI_USERNAME | ISCSI_PASSWORD |
++ ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
++ ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
++ ISCSI_LU_RESET_TMO |
++ ISCSI_PING_TMO | ISCSI_RECV_TMO |
++ ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
++ .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
++ ISCSI_HOST_INITIATOR_NAME | ISCSI_HOST_NETDEV_NAME,
++ .get_host_param = cxgb3i_host_get_param,
++ .set_host_param = cxgb3i_host_set_param,
++ /* session management */
++ .create_session = cxgb3i_session_create,
++ .destroy_session = cxgb3i_session_destroy,
++ .get_session_param = iscsi_session_get_param,
++ /* connection management */
++ .create_conn = cxgb3i_conn_create,
++ .bind_conn = cxgb3i_conn_bind,
++ .destroy_conn = iscsi_conn_teardown,
++ .start_conn = iscsi_conn_start,
++ .stop_conn = iscsi_conn_stop,
++ .get_conn_param = cxgb3i_conn_get_param,
++ .set_param = cxgb3i_conn_set_param,
++ .get_stats = cxgb3i_conn_get_stats,
++ /* pdu xmit req. from user space */
++ .send_pdu = iscsi_conn_send_pdu,
++ /* task */
++ .init_task = iscsi_tcp_task_init,
++ .xmit_task = iscsi_tcp_task_xmit,
++ .cleanup_task = iscsi_tcp_cleanup_task,
++ .parse_itt = cxgb3i_parse_itt,
++ .reserve_itt = cxgb3i_reserve_itt,
++ .release_itt = cxgb3i_release_itt,
++ /* TCP connect/disconnect */
++ .ep_connect = cxgb3i_ep_connect,
++ .ep_poll = cxgb3i_ep_poll,
++ .ep_disconnect = cxgb3i_ep_disconnect,
++ /* Error recovery timeout call */
++ .session_recovery_timedout = iscsi_session_recovery_timedout,
++};
++
++int cxgb3i_iscsi_init(void)
++{
++ cxgb3i_scsi_transport =
++ iscsi_register_transport(&cxgb3i_iscsi_transport);
++ if (!cxgb3i_scsi_transport) {
++ cxgb3i_log_error("Could not register cxgb3i transport.\n");
++ return -ENODEV;
++ }
++ cxgb3i_log_debug("cxgb3i transport 0x%p.\n", cxgb3i_scsi_transport);
++ return 0;
++}
++
++void cxgb3i_iscsi_cleanup(void)
++{
++ if (cxgb3i_scsi_transport) {
++ cxgb3i_log_debug("cxgb3i transport 0x%p.\n",
++ cxgb3i_scsi_transport);
++ iscsi_unregister_transport(&cxgb3i_iscsi_transport);
++ }
++}
+--- /dev/null
++++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
+@@ -0,0 +1,2021 @@
++/*
++ * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
++ *
++ * Written by Dimitris Michailidis (dm@chelsio.com)
++ *
++ * This program is distributed in the hope that it will be useful, but WITHOUT
++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
++ * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
++ * release for licensing terms and conditions.
++ */
++
++#include <linux/if_vlan.h>
++#include <linux/version.h>
++
++#include "cxgb3_defs.h"
++#include "cxgb3_ctl_defs.h"
++#include "firmware_exports.h"
++#include "cxgb3i_offload.h"
++#include "cxgb3i_ulp2.h"
++
++static int cxgb3_rcv_win = 256 * 1024;
++module_param(cxgb3_rcv_win, int, 0644);
++MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
++
++static int cxgb3_snd_win = 64 * 1024;
++module_param(cxgb3_snd_win, int, 0644);
++MODULE_PARM_DESC(cxgb3_snd_win, "TCP send window in bytes (default=64KB)");
++
++static int cxgb3_rx_credit_thres = 10 * 1024;
++module_param(cxgb3_rx_credit_thres, int, 0644);
++MODULE_PARM_DESC(rx_credit_thres,
++ "RX credits return threshold in bytes (default=10KB)");
++
++static unsigned int cxgb3_max_connect = 8 * 1024;
++module_param(cxgb3_max_connect, uint, 0644);
++MODULE_PARM_DESC(cxgb3_max_connect, "Max. # of connections (default=8092)");
++
++static unsigned int cxgb3_sport_base = 20000;
++module_param(cxgb3_sport_base, uint, 0644);
++MODULE_PARM_DESC(cxgb3_sport_base, "starting port number (default=20000)");
++
++#ifdef __DEBUG_C3CN_TX__
++#define c3cn_tx_debug cxgb3i_log_debug
++#else
++#define c3cn_tx_debug(fmt...)
++#endif
++
++/* connection flags */
++static inline void c3cn_set_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
++{
++ __set_bit(flag, &c3cn->flags);
++ c3cn_conn_debug("c3cn 0x%p, set %d, s 0x%x, f 0x%lx.\n",
++ c3cn, flag, c3cn->state, c3cn->flags);
++}
++
++static inline void c3cn_reset_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
++{
++ __clear_bit(flag, &c3cn->flags);
++ c3cn_conn_debug("c3cn 0x%p, clear %d, s 0x%x, f 0x%lx.\n",
++ c3cn, flag, c3cn->state, c3cn->flags);
++}
++
++static inline int c3cn_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
++{
++ if (c3cn == NULL)
++ return 0;
++ return test_bit(flag, &c3cn->flags);
++}
++
++/* connection state */
++static void c3cn_set_state(struct s3_conn *c3cn, int state)
++{
++ c3cn_conn_debug("c3cn 0x%p state -> 0x%x.\n", c3cn, state);
++ c3cn->state = state;
++}
++
++/* connection reference count */
++static inline void c3cn_hold(struct s3_conn *c3cn)
++{
++ atomic_inc(&c3cn->refcnt);
++}
++
++static inline void c3cn_put(struct s3_conn *c3cn)
++{
++ if (atomic_dec_and_test(&c3cn->refcnt)) {
++ c3cn_conn_debug("free c3cn 0x%p, 0x%x, 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++ kfree(c3cn);
++ }
++}
++
++/* minimal port allocation management scheme */
++static spinlock_t sport_map_lock;
++static unsigned int sport_map_next;
++static unsigned long *sport_map;
++
++/*
++ * Find a free source port in our allocation map. We use a very simple rotor
++ * scheme to look for the next free port.
++ *
++ * If a source port has been specified make sure that it doesn't collide with
++ * our normal source port allocation map. If it's outside the range of our
++ * allocation scheme just let them use it.
++ */
++static int c3cn_get_port(struct s3_conn *c3cn)
++{
++ unsigned int start;
++
++ if (!sport_map)
++ goto error_out;
++
++ if (c3cn->saddr.sin_port != 0) {
++ int sport = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
++ int err = 0;
++
++ if (sport < 0 || sport >= cxgb3_max_connect)
++ return 0;
++ spin_lock(&sport_map_lock);
++ err = __test_and_set_bit(sport, sport_map);
++ spin_unlock(&sport_map_lock);
++ return err ? -EADDRINUSE : 0;
++ }
++
++ spin_lock(&sport_map_lock);
++ start = sport_map_next;
++ do {
++ unsigned int new = sport_map_next;
++ if (++sport_map_next >= cxgb3_max_connect)
++ sport_map_next = 0;
++ if (!(__test_and_set_bit(new, sport_map))) {
++ spin_unlock(&sport_map_lock);
++ c3cn_conn_debug("reserve port %u.\n",
++ cxgb3_sport_base + new);
++ c3cn->saddr.sin_port = htons(cxgb3_sport_base + new);
++ return 0;
++ }
++ } while (sport_map_next != start);
++ spin_unlock(&sport_map_lock);
++
++error_out:
++ return -EADDRNOTAVAIL;
++}
++
++/*
++ * Deallocate a source port from the allocation map. If the source port is
++ * outside our allocation range just return -- the caller is responsible for
++ * keeping track of their port usage outside of our allocation map.
++ */
++static void c3cn_put_port(struct s3_conn *c3cn)
++{
++ if (c3cn->saddr.sin_port) {
++ int old = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
++ c3cn->saddr.sin_port = 0;
++
++ if (old < 0 || old >= cxgb3_max_connect)
++ return;
++
++ c3cn_conn_debug("release port %u.\n", cxgb3_sport_base + old);
++ spin_lock(&sport_map_lock);
++ __clear_bit(old, sport_map);
++ spin_unlock(&sport_map_lock);
++ }
++}
++
++static void c3cn_reset_timer(struct s3_conn *c3cn, struct timer_list *timer,
++ unsigned long expires)
++{
++ if (!mod_timer(timer, expires))
++ c3cn_hold(c3cn);
++}
++
++typedef int (cxgb3_cpl_handler_decl) (struct t3cdev *,
++ struct sk_buff *, void *);
++
++static cxgb3_cpl_handler_decl do_act_establish;
++static cxgb3_cpl_handler_decl do_act_open_rpl;
++static cxgb3_cpl_handler_decl do_wr_ack;
++static cxgb3_cpl_handler_decl do_peer_close;
++static cxgb3_cpl_handler_decl do_abort_req;
++static cxgb3_cpl_handler_decl do_abort_rpl;
++static cxgb3_cpl_handler_decl do_close_con_rpl;
++static cxgb3_cpl_handler_decl do_iscsi_hdr;
++
++static LIST_HEAD(cxgb3_list);
++static DEFINE_MUTEX(cxgb3_list_lock);
++
++/*
++ * For ULP connections HW may inserts digest bytes into the pdu. This array
++ * contains the compensating extra lengths for ULP packets. It is indexed by
++ * a packet's ULP submode.
++ */
++static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 };
++
++/*
++ * Return the length of any HW additions that will be made to a Tx packet.
++ * Such additions can happen for some types of ULP packets.
++ */
++static inline unsigned int ulp_extra_len(const struct sk_buff *skb)
++{
++ return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3];
++}
++
++/*
++ * Size of WRs in bytes. Note that we assume all devices we are handling have
++ * the same WR size.
++ */
++static unsigned int wrlen __read_mostly;
++
++/*
++ * The number of WRs needed for an skb depends on the number of page fragments
++ * in the skb and whether it has any payload in its main body. This maps the
++ * length of the gather list represented by an skb into the # of necessary WRs.
++ */
++static unsigned int skb_wrs[MAX_SKB_FRAGS + 2] __read_mostly;
++
++static void s3_init_wr_tab(unsigned int wr_len)
++{
++ int i;
++
++ if (skb_wrs[1]) /* already initialized */
++ return;
++
++ for (i = 1; i < ARRAY_SIZE(skb_wrs); i++) {
++ int sgl_len = (3 * i) / 2 + (i & 1);
++
++ sgl_len += 3;
++ skb_wrs[i] = (sgl_len <= wr_len
++ ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
++ }
++
++ wrlen = wr_len * 8;
++}
++
++/*
++ * cxgb3i API operations.
++ */
++/*
++ * large memory chunk allocation/release
++ */
++void *cxgb3i_alloc_big_mem(unsigned int size)
++{
++ void *p = kmalloc(size, GFP_KERNEL);
++ if (!p)
++ p = vmalloc(size);
++ if (p)
++ memset(p, 0, size);
++ return p;
++}
++
++void cxgb3i_free_big_mem(void *addr)
++{
++ if (is_vmalloc_addr(addr))
++ vfree(addr);
++ else
++ kfree(addr);
++}
++
++void cxgb3i_sdev_cleanup(void)
++{
++ if (sport_map)
++ cxgb3i_free_big_mem(sport_map);
++}
++
++int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers)
++{
++ cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish;
++ cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl;
++ cpl_handlers[CPL_PEER_CLOSE] = do_peer_close;
++ cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req;
++ cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl;
++ cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl;
++ cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack;
++ cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr;
++
++ if (cxgb3_max_connect > CXGB3I_MAX_CONN)
++ cxgb3_max_connect = CXGB3I_MAX_CONN;
++ sport_map = cxgb3i_alloc_big_mem(DIV_ROUND_UP(cxgb3_max_connect,
++ 8 *
++ sizeof(unsigned long)));
++ if (!sport_map)
++ return -ENOMEM;
++ return 0;
++}
++
++void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client)
++{
++ struct cxgb3i_sdev_data *cdata;
++ struct adap_ports *ports;
++ struct ofld_page_info rx_page_info;
++ unsigned int wr_len;
++ int i;
++
++ cdata = kzalloc(sizeof *cdata, GFP_KERNEL);
++ if (!cdata)
++ return;
++ ports = kzalloc(sizeof *ports, GFP_KERNEL);
++ if (!ports)
++ goto free_ports;
++ cdata->ports = ports;
++
++ if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 ||
++ cdev->ctl(cdev, GET_PORTS, cdata->ports) < 0 ||
++ cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0)
++ goto free_ports;
++
++ s3_init_wr_tab(wr_len);
++
++ INIT_LIST_HEAD(&cdata->list);
++ cdata->cdev = cdev;
++ cdata->client = client;
++ cdata->rx_page_size = rx_page_info.page_size;
++ skb_queue_head_init(&cdata->deferq);
++
++ for (i = 0; i < ports->nports; i++)
++ NDEV2CDATA(ports->lldevs[i]) = cdata;
++
++ mutex_lock(&cxgb3_list_lock);
++ list_add_tail(&cdata->list, &cxgb3_list);
++ mutex_unlock(&cxgb3_list_lock);
++
++ return;
++
++free_ports:
++ kfree(ports);
++ kfree(cdata);
++}
++
++void cxgb3i_sdev_remove(struct t3cdev *cdev)
++{
++ struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
++ struct adap_ports *ports = cdata->ports;
++ int i;
++
++ for (i = 0; i < ports->nports; i++)
++ NDEV2CDATA(ports->lldevs[i]) = NULL;
++
++ mutex_lock(&cxgb3_list_lock);
++ list_del(&cdata->list);
++ mutex_unlock(&cxgb3_list_lock);
++
++ kfree(ports);
++ kfree(cdata);
++}
++
++/*
++ * Return TRUE if the specified net device is for a port on one of our
++ * registered adapters.
++ */
++static int is_cxgb3_dev(struct net_device *dev)
++{
++ struct cxgb3i_sdev_data *cdata;
++
++ mutex_lock(&cxgb3_list_lock);
++ list_for_each_entry(cdata, &cxgb3_list, list) {
++ struct adap_ports *ports = cdata->ports;
++ int i;
++
++ for (i = 0; i < ports->nports; i++)
++ if (dev == ports->lldevs[i]) {
++ mutex_unlock(&cxgb3_list_lock);
++ return 1;
++ }
++ }
++ mutex_unlock(&cxgb3_list_lock);
++ return 0;
++}
++
++/*
++ * Primary cxgb3 API operations.
++ * =============================
++ */
++
++static int s3_push_frames(struct s3_conn *, int);
++static int s3_send_reset(struct s3_conn *, int, struct sk_buff *);
++static void t3_release_offload_resources(struct s3_conn *);
++static void mk_close_req(struct s3_conn *);
++
++struct s3_conn *cxgb3i_c3cn_create(void)
++{
++ struct s3_conn *c3cn;
++
++ c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL);
++ if (c3cn == NULL)
++ return NULL;
++
++ c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn);
++
++ c3cn->flags = 0;
++ spin_lock_init(&c3cn->lock);
++ atomic_set(&c3cn->refcnt, 1);
++ skb_queue_head_init(&c3cn->receive_queue);
++ skb_queue_head_init(&c3cn->write_queue);
++ setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn);
++ rwlock_init(&c3cn->callback_lock);
++
++ return c3cn;
++}
++
++static inline void s3_purge_write_queue(struct s3_conn *c3cn)
++{
++ struct sk_buff *skb;
++
++ while ((skb = __skb_dequeue(&c3cn->write_queue)))
++ __kfree_skb(skb);
++}
++
++static void c3cn_done(struct s3_conn *c3cn)
++{
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ c3cn_put_port(c3cn);
++ t3_release_offload_resources(c3cn);
++ c3cn_set_state(c3cn, C3CN_STATE_CLOSE);
++ c3cn->shutdown = C3CN_SHUTDOWN_MASK;
++ cxgb3i_conn_closing(c3cn);
++}
++
++static void c3cn_close(struct s3_conn *c3cn)
++{
++ int data_lost, old_state;
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ dst_confirm(c3cn->dst_cache);
++
++ spin_lock_bh(&c3cn->lock);
++ c3cn->shutdown |= C3CN_SHUTDOWN_MASK;
++
++ /*
++ * We need to flush the receive buffs. We do this only on the
++ * descriptor close, not protocol-sourced closes, because the
++ * reader process may not have drained the data yet! Make a note
++ * of whether any received data will be lost so we can decide whether
++ * to FIN or RST.
++ */
++ data_lost = skb_queue_len(&c3cn->receive_queue);
++ __skb_queue_purge(&c3cn->receive_queue);
++
++ if (c3cn->state == C3CN_STATE_CLOSE)
++ /* Nothing if we are already closed */
++ c3cn_conn_debug("c3cn 0x%p, 0x%x, already closed.\n",
++ c3cn, c3cn->state);
++ else if (data_lost || c3cn->state == C3CN_STATE_SYN_SENT) {
++ c3cn_conn_debug("c3cn 0x%p, 0x%x -> closing, send reset.\n",
++ c3cn, c3cn->state);
++ /* Unread data was tossed, zap the connection. */
++ s3_send_reset(c3cn, CPL_ABORT_SEND_RST, NULL);
++ goto unlock;
++ } else if (c3cn->state == C3CN_STATE_ESTABLISHED) {
++ c3cn_conn_debug("c3cn 0x%p, est. -> closing, send close_req.\n",
++ c3cn);
++ c3cn_set_state(c3cn, C3CN_STATE_CLOSING);
++ mk_close_req(c3cn);
++ }
++
++unlock:
++ old_state = c3cn->state;
++ c3cn_hold(c3cn); /* must last past the potential destroy() */
++
++ spin_unlock_bh(&c3cn->lock);
++
++ /*
++ * There are no more user references at this point. Grab the
++ * connection lock and finish the close.
++ */
++ local_bh_disable();
++ spin_lock(&c3cn->lock);
++
++ /*
++ * Because the connection was orphaned before the spin_lock()
++ * either the backlog or a BH may have already destroyed it.
++ * Bail out if so.
++ */
++ if (old_state != C3CN_STATE_CLOSE && c3cn->state == C3CN_STATE_CLOSE)
++ goto out;
++
++ if (c3cn->state == C3CN_STATE_CLOSE)
++ s3_purge_write_queue(c3cn);
++
++out:
++ spin_unlock(&c3cn->lock);
++ local_bh_enable();
++ c3cn_put(c3cn);
++}
++
++void cxgb3i_c3cn_release(struct s3_conn *c3cn)
++{
++ c3cn_conn_debug("c3cn 0x%p, s 0x%x, f 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++ if (likely(c3cn->state != C3CN_STATE_SYN_SENT))
++ c3cn_close(c3cn);
++ else
++ c3cn_set_flag(c3cn, C3CN_CLOSE_NEEDED);
++ c3cn_put(c3cn);
++}
++
++
++/*
++ * Local utility routines used to implement primary cxgb3 API operations.
++ * ======================================================================
++ */
++
++static u32 s3_send_rx_credits(struct s3_conn *, u32, u32, int);
++static int act_open(struct s3_conn *, struct net_device *);
++static void mk_act_open_req(struct s3_conn *, struct sk_buff *,
++ unsigned int, const struct l2t_entry *);
++static void skb_entail(struct s3_conn *, struct sk_buff *, int);
++
++static inline void reset_wr_list(struct s3_conn *c3cn)
++{
++ c3cn->wr_pending_head = NULL;
++}
++
++/*
++ * Add a WR to a connections's list of pending WRs. This is a singly-linked
++ * list of sk_buffs operating as a FIFO. The head is kept in wr_pending_head
++ * and the tail in wr_pending_tail.
++ */
++static inline void enqueue_wr(struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ skb->sp = NULL;
++
++ /*
++ * We want to take an extra reference since both us and the driver
++ * need to free the packet before it's really freed. We know there's
++ * just one user currently so we use atomic_set rather than skb_get
++ * to avoid the atomic op.
++ */
++ atomic_set(&skb->users, 2);
++
++ if (!c3cn->wr_pending_head)
++ c3cn->wr_pending_head = skb;
++ else
++ c3cn->wr_pending_tail->sp = (void *)skb;
++ c3cn->wr_pending_tail = skb;
++}
++
++/*
++ * The next two functions calculate the option 0 value for a connection.
++ */
++static inline int compute_wscale(int win)
++{
++ int wscale = 0;
++ while (wscale < 14 && (65535<<wscale) < win)
++ wscale++;
++ return wscale;
++}
++
++static inline unsigned int calc_opt0h(struct s3_conn *c3cn)
++{
++ int wscale = compute_wscale(cxgb3_rcv_win);
++ return V_KEEP_ALIVE(1) |
++ F_TCAM_BYPASS |
++ V_WND_SCALE(wscale) |
++ V_MSS_IDX(c3cn->mss_idx);
++}
++
++static inline unsigned int calc_opt0l(struct s3_conn *c3cn)
++{
++ return V_ULP_MODE(ULP_MODE_ISCSI) |
++ V_RCV_BUFSIZ(cxgb3_rcv_win>>10);
++}
++
++static inline void make_tx_data_wr(struct s3_conn *c3cn,
++ struct sk_buff *skb, int len)
++{
++ struct tx_data_wr *req;
++
++ skb_reset_transport_header(skb);
++ req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
++ req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
++ req->wr_lo = htonl(V_WR_TID(c3cn->tid));
++ req->sndseq = htonl(c3cn->snd_nxt);
++ /* len includes the length of any HW ULP additions */
++ req->len = htonl(len);
++ req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx));
++ /* V_TX_ULP_SUBMODE sets both the mode and submode */
++ req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) |
++ V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
++
++ if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
++
++ req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
++ V_TX_CPU_IDX(c3cn->qset));
++
++ /* Sendbuffer is in units of 32KB.
++ */
++ req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
++ c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
++ }
++}
++
++/**
++ * cxgb3_egress_dev - return the cxgb3 egress device
++ * @root_dev: the root device anchoring the search
++ * @c3cn: the connection used to determine egress port in bonding mode
++ * @context: in bonding mode, indicates a connection set up or failover
++ *
++ * Return egress device or NULL if the egress device isn't one of our ports.
++ *
++ * Given a root network device it returns the physical egress device that is a
++ * descendant of the root device. The root device may be either a physical
++ * device, in which case it is the device returned, or a virtual device, such
++ * as a VLAN or bonding device. In case of a bonding device the search
++ * considers the decisions of the bonding device given its mode to locate the
++ * correct egress device.
++ */
++static struct net_device *cxgb3_egress_dev(struct net_device *root_dev,
++ struct s3_conn *c3cn,
++ int context)
++{
++ while (root_dev) {
++ if (root_dev->priv_flags & IFF_802_1Q_VLAN)
++ root_dev = vlan_dev_real_dev(root_dev);
++ else if (is_cxgb3_dev(root_dev))
++ return root_dev;
++ else
++ return NULL;
++ }
++ return NULL;
++}
++
++static struct rtable *find_route(__be32 saddr, __be32 daddr,
++ __be16 sport, __be16 dport)
++{
++ struct rtable *rt;
++ struct flowi fl = {
++ .oif = 0,
++ .nl_u = {
++ .ip4_u = {
++ .daddr = daddr,
++ .saddr = saddr,
++ .tos = 0 } },
++ .proto = IPPROTO_TCP,
++ .uli_u = {
++ .ports = {
++ .sport = sport,
++ .dport = dport } } };
++
++ if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
++ return NULL;
++ return rt;
++}
++
++int cxgb3i_c3cn_connect(struct s3_conn *c3cn, struct sockaddr_in *usin)
++{
++ struct rtable *rt;
++ struct net_device *dev;
++ struct cxgb3i_sdev_data *cdata;
++ struct t3cdev *cdev;
++ __be32 sipv4;
++ int err;
++
++ if (usin->sin_family != AF_INET)
++ return -EAFNOSUPPORT;
++
++ /* get a source port if one hasn't been provided */
++ err = c3cn_get_port(c3cn);
++ if (err)
++ return err;
++
++ c3cn_conn_debug("c3cn 0x%p get port %u.\n",
++ c3cn, ntohs(c3cn->saddr.sin_port));
++
++ c3cn->daddr.sin_port = usin->sin_port;
++ c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr;
++
++ rt = find_route(c3cn->saddr.sin_addr.s_addr,
++ c3cn->daddr.sin_addr.s_addr,
++ c3cn->saddr.sin_port,
++ c3cn->daddr.sin_port);
++ if (rt == NULL) {
++ c3cn_conn_debug("NO route to 0x%x, port %u.\n",
++ c3cn->daddr.sin_addr.s_addr,
++ ntohs(c3cn->daddr.sin_port));
++ return -ENETUNREACH;
++ }
++
++ if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
++ c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n",
++ c3cn->daddr.sin_addr.s_addr,
++ ntohs(c3cn->daddr.sin_port));
++ ip_rt_put(rt);
++ return -ENETUNREACH;
++ }
++
++ if (!c3cn->saddr.sin_addr.s_addr)
++ c3cn->saddr.sin_addr.s_addr = rt->rt_src;
++
++ /* now commit destination to connection */
++ c3cn->dst_cache = &rt->u.dst;
++
++ /* try to establish an offloaded connection */
++ dev = cxgb3_egress_dev(c3cn->dst_cache->dev, c3cn, 0);
++ if (dev == NULL) {
++ c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn);
++ return -ENETUNREACH;
++ }
++ cdata = NDEV2CDATA(dev);
++ cdev = cdata->cdev;
++
++ sipv4 = cxgb3i_get_private_ipv4addr(dev);
++ if (!sipv4) {
++ c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn);
++ sipv4 = c3cn->saddr.sin_addr.s_addr;
++ cxgb3i_set_private_ipv4addr(dev, sipv4);
++ } else
++ c3cn->saddr.sin_addr.s_addr = sipv4;
++
++ c3cn_conn_debug("c3cn 0x%p, %u.%u.%u.%u,%u-%u.%u.%u.%u,%u SYN_SENT.\n",
++ c3cn, NIPQUAD(c3cn->saddr.sin_addr.s_addr),
++ ntohs(c3cn->saddr.sin_port),
++ NIPQUAD(c3cn->daddr.sin_addr.s_addr),
++ ntohs(c3cn->daddr.sin_port));
++
++ c3cn_set_state(c3cn, C3CN_STATE_SYN_SENT);
++
++ if (!act_open(c3cn, dev))
++ return 0;
++
++ /*
++ * If we get here, we don't have an offload connection so simply
++ * return a failure.
++ */
++ err = -ENOTSUPP;
++
++ /*
++ * This trashes the connection and releases the local port,
++ * if necessary.
++ */
++ c3cn_conn_debug("c3cn 0x%p -> CLOSE.\n", c3cn);
++ c3cn_set_state(c3cn, C3CN_STATE_CLOSE);
++ ip_rt_put(rt);
++ c3cn_put_port(c3cn);
++ c3cn->daddr.sin_port = 0;
++ return err;
++}
++
++/*
++ * Set of states for which we should return RX credits.
++ */
++#define CREDIT_RETURN_STATE (C3CN_STATE_ESTABLISHED)
++
++/*
++ * Called after some received data has been read. It returns RX credits
++ * to the HW for the amount of data processed.
++ */
++void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied)
++{
++ struct t3cdev *cdev;
++ int must_send;
++ u32 credits, dack = 0;
++
++ if (!c3cn_in_state(c3cn, CREDIT_RETURN_STATE))
++ return;
++
++ credits = c3cn->copied_seq - c3cn->rcv_wup;
++ if (unlikely(!credits))
++ return;
++
++ cdev = c3cn->cdev;
++
++ if (unlikely(cxgb3_rx_credit_thres == 0))
++ return;
++
++ dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
++
++ /*
++ * For coalescing to work effectively ensure the receive window has
++ * at least 16KB left.
++ */
++ must_send = credits + 16384 >= cxgb3_rcv_win;
++
++ if (must_send || credits >= cxgb3_rx_credit_thres)
++ c3cn->rcv_wup += s3_send_rx_credits(c3cn, credits, dack,
++ must_send);
++}
++
++/*
++ * Generic ARP failure handler that discards the buffer.
++ */
++static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb)
++{
++ kfree_skb(skb);
++}
++
++/*
++ * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
++ * connection's send queue and sends them on to T3. Must be called with the
++ * connection's lock held. Returns the amount of send buffer space that was
++ * freed as a result of sending queued data to T3.
++ */
++static int s3_push_frames(struct s3_conn *c3cn, int req_completion)
++{
++ int total_size = 0;
++ struct sk_buff *skb;
++ struct t3cdev *cdev;
++ struct cxgb3i_sdev_data *cdata;
++
++ if (unlikely(c3cn_in_state(c3cn,
++ C3CN_STATE_SYN_SENT | C3CN_STATE_CLOSE)))
++ return 0;
++
++ /*
++ * We shouldn't really be called at all after an abort but check just
++ * in case.
++ */
++ if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN)))
++ return 0;
++
++ cdev = c3cn->cdev;
++ cdata = CXGB3_SDEV_DATA(cdev);
++
++ while (c3cn->wr_avail
++ && (skb = skb_peek(&c3cn->write_queue)) != NULL
++ && !c3cn_flag(c3cn, C3CN_TX_WAIT_IDLE)) {
++
++ int len = skb->len; /* length before skb_push */
++ int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
++ int wrs_needed = skb_wrs[frags];
++
++ if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
++ wrs_needed = 1;
++
++ WARN_ON(frags >= ARRAY_SIZE(skb_wrs) || wrs_needed < 1);
++
++ if (c3cn->wr_avail < wrs_needed)
++ break;
++
++ __skb_unlink(skb, &c3cn->write_queue);
++ skb->priority = CPL_PRIORITY_DATA;
++ skb->csum = wrs_needed; /* remember this until the WR_ACK */
++ c3cn->wr_avail -= wrs_needed;
++ c3cn->wr_unacked += wrs_needed;
++ enqueue_wr(c3cn, skb);
++
++ if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
++ len += ulp_extra_len(skb);
++ make_tx_data_wr(c3cn, skb, len);
++ c3cn->snd_nxt += len;
++ if ((req_completion
++ && c3cn->wr_unacked == wrs_needed)
++ || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL)
++ || c3cn->wr_unacked >= c3cn->wr_max / 2) {
++ struct work_request_hdr *wr = cplhdr(skb);
++
++ wr->wr_hi |= htonl(F_WR_COMPL);
++ c3cn->wr_unacked = 0;
++ }
++ CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR;
++ } else if (skb->data[0] == FW_WROPCODE_OFLD_CLOSE_CON)
++ c3cn_set_flag(c3cn, C3CN_CLOSE_CON_REQUESTED);
++
++ total_size += skb->truesize;
++ set_arp_failure_handler(skb, arp_failure_discard);
++ l2t_send(cdev, skb, c3cn->l2t);
++ }
++ return total_size;
++}
++
++/*
++ * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant
++ * and send it along.
++ */
++static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb)
++{
++ struct cpl_abort_req *req = cplhdr(skb);
++
++ c3cn_conn_debug("tdev 0x%p.\n", cdev);
++
++ req->cmd = CPL_ABORT_NO_RST;
++ cxgb3_ofld_send(cdev, skb);
++}
++
++/*
++ * Send an ABORT_REQ message. Cannot fail. This routine makes sure we do
++ * not send multiple ABORT_REQs for the same connection and also that we do
++ * not try to send a message after the connection has closed. Returns 1 if
++ * an ABORT_REQ wasn't generated after all, 0 otherwise.
++ */
++static int s3_send_reset(struct s3_conn *c3cn, int mode,
++ struct sk_buff *skb)
++{
++ struct cpl_abort_req *req;
++ unsigned int tid = c3cn->tid;
++
++ if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN) || !c3cn->cdev)) {
++ if (skb)
++ __kfree_skb(skb);
++ return 1;
++ }
++
++ c3cn_conn_debug("c3cn 0x%p, mode %d, flag ABORT_RPL + ABORT_SHUT.\n",
++ c3cn, mode);
++
++ c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING);
++ c3cn_set_flag(c3cn, C3CN_ABORT_SHUTDOWN);
++
++ /* Purge the send queue so we don't send anything after an abort. */
++ s3_purge_write_queue(c3cn);
++
++ if (!skb)
++ skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
++ skb->priority = CPL_PRIORITY_DATA;
++ set_arp_failure_handler(skb, abort_arp_failure);
++
++ req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
++ req->wr.wr_lo = htonl(V_WR_TID(tid));
++ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
++ req->rsvd0 = htonl(c3cn->snd_nxt);
++ req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT);
++ req->cmd = mode;
++
++ l2t_send(c3cn->cdev, skb, c3cn->l2t);
++ return 0;
++}
++
++/*
++ * Add a list of skbs to a connection send queue. This interface is intended
++ * for use by in-kernel ULPs. The skbs must comply with the max size limit of
++ * the device and have a headroom of at least TX_HEADER_LEN bytes.
++ */
++int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb, int flags)
++{
++ struct sk_buff *next;
++ int err, copied = 0;
++
++ spin_lock_bh(&c3cn->lock);
++
++ if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED)) {
++ err = -EAGAIN;
++ goto out_err;
++ }
++
++ err = -EPIPE;
++ if (c3cn->err || (c3cn->shutdown & C3CN_SEND_SHUTDOWN))
++ goto out_err;
++
++ while (skb) {
++ if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
++ c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
++ err = -EINVAL;
++ goto out_err;
++ }
++
++ next = skb->next;
++ skb->next = NULL;
++ skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
++ copied += skb->len;
++ c3cn->write_seq += skb->len + ulp_extra_len(skb);
++ skb = next;
++ }
++done:
++ if (likely(skb_queue_len(&c3cn->write_queue)))
++ s3_push_frames(c3cn, 1);
++ spin_unlock_bh(&c3cn->lock);
++ return copied;
++
++out_err:
++ if (copied == 0 && err == -EPIPE)
++ copied = c3cn->err ? c3cn->err : -EPIPE;
++ goto done;
++}
++
++/*
++ * Low-level utility routines for primary API functions.
++ * =====================================================
++ */
++/* routines to implement CPL message processing */
++static void c3cn_act_establish(struct s3_conn *, struct sk_buff *);
++static void active_open_failed(struct s3_conn *, struct sk_buff *);
++static void wr_ack(struct s3_conn *, struct sk_buff *);
++static void do_peer_fin(struct s3_conn *, struct sk_buff *);
++static void process_abort_req(struct s3_conn *, struct sk_buff *);
++static void process_abort_rpl(struct s3_conn *, struct sk_buff *);
++static void process_close_con_rpl(struct s3_conn *, struct sk_buff *);
++static void process_rx_iscsi_hdr(struct s3_conn *, struct sk_buff *);
++
++static struct sk_buff *__get_cpl_reply_skb(struct sk_buff *, size_t, gfp_t);
++
++static void fail_act_open(struct s3_conn *, int);
++static void init_offload_conn(struct s3_conn *, struct t3cdev *,
++ struct dst_entry *);
++
++/*
++ * Insert a connection into the TID table and take an extra reference.
++ */
++static inline void c3cn_insert_tid(struct cxgb3i_sdev_data *cdata,
++ struct s3_conn *c3cn,
++ unsigned int tid)
++{
++ c3cn_hold(c3cn);
++ cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid);
++}
++
++static inline void free_atid(struct t3cdev *cdev, unsigned int tid)
++{
++ struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid);
++ if (c3cn)
++ c3cn_put(c3cn);
++}
++
++/*
++ * This function is intended for allocations of small control messages.
++ * Such messages go as immediate data and usually the pakets are freed
++ * immediately. We maintain a cache of one small sk_buff and use it whenever
++ * it is available (has a user count of 1). Otherwise we get a fresh buffer.
++ */
++#define CTRL_SKB_LEN 120
++
++static struct sk_buff *alloc_ctrl_skb(const struct s3_conn *c3cn,
++ int len)
++{
++ struct sk_buff *skb = c3cn->ctrl_skb_cache;
++
++ if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
++ __skb_trim(skb, 0);
++ atomic_set(&skb->users, 2);
++ } else if (likely(!in_atomic()))
++ skb = alloc_skb(len, GFP_ATOMIC | __GFP_NOFAIL);
++ else
++ skb = alloc_skb(len, GFP_ATOMIC);
++ return skb;
++}
++
++/*
++ * Handle an ARP failure for an active open.
++ */
++static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
++{
++ struct s3_conn *c3cn = (struct s3_conn *)skb->sk;
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
++
++ c3cn_hold(c3cn);
++ spin_lock(&c3cn->lock);
++ if (c3cn->state == C3CN_STATE_SYN_SENT) {
++ fail_act_open(c3cn, EHOSTUNREACH);
++ __kfree_skb(skb);
++ }
++ spin_unlock(&c3cn->lock);
++ c3cn_put(c3cn);
++}
++
++/*
++ * Send an active open request.
++ */
++static int act_open(struct s3_conn *c3cn, struct net_device *dev)
++{
++ struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev);
++ struct t3cdev *cdev = cdata->cdev;
++ struct dst_entry *dst = c3cn->dst_cache;
++ struct sk_buff *skb;
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++ /*
++ * Initialize connection data. Note that the flags and ULP mode are
++ * initialized higher up ...
++ */
++ c3cn->dev = dev;
++ c3cn->cdev = cdev;
++ c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn);
++ if (c3cn->tid < 0)
++ goto out_err;
++
++ c3cn->qset = 0;
++ c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev);
++ if (!c3cn->l2t)
++ goto free_tid;
++
++ skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL);
++ if (!skb)
++ goto free_l2t;
++
++ skb->sk = (struct sock *)c3cn;
++ set_arp_failure_handler(skb, act_open_req_arp_failure);
++
++ c3cn_hold(c3cn);
++
++ init_offload_conn(c3cn, cdev, dst);
++ c3cn->err = 0;
++ c3cn_reset_flag(c3cn, C3CN_DONE);
++
++ mk_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
++ l2t_send(cdev, skb, c3cn->l2t);
++ return 0;
++
++free_l2t:
++ l2t_release(L2DATA(cdev), c3cn->l2t);
++free_tid:
++ free_atid(cdev, c3cn->tid);
++ c3cn->tid = 0;
++out_err:
++ return -1;
++}
++
++/*
++ * Close a connection by sending a CPL_CLOSE_CON_REQ message. Cannot fail
++ * under any circumstances. We take the easy way out and always queue the
++ * message to the write_queue. We can optimize the case where the queue is
++ * already empty though the optimization is probably not worth it.
++ */
++static void mk_close_req(struct s3_conn *c3cn)
++{
++ struct sk_buff *skb;
++ struct cpl_close_con_req *req;
++ unsigned int tid = c3cn->tid;
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ skb = alloc_skb(sizeof(struct cpl_close_con_req),
++ GFP_KERNEL | __GFP_NOFAIL);
++ req = (struct cpl_close_con_req *)__skb_put(skb, sizeof(*req));
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
++ req->wr.wr_lo = htonl(V_WR_TID(tid));
++ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
++ req->rsvd = htonl(c3cn->write_seq);
++
++ skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND);
++ if (c3cn->state != C3CN_STATE_SYN_SENT)
++ s3_push_frames(c3cn, 1);
++}
++
++static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
++ int flags)
++{
++ CXGB3_SKB_CB(skb)->seq = c3cn->write_seq;
++ CXGB3_SKB_CB(skb)->flags = flags;
++ __skb_queue_tail(&c3cn->write_queue, skb);
++}
++
++/*
++ * Send RX credits through an RX_DATA_ACK CPL message. If nofail is 0 we are
++ * permitted to return without sending the message in case we cannot allocate
++ * an sk_buff. Returns the number of credits sent.
++ */
++static u32 s3_send_rx_credits(struct s3_conn *c3cn, u32 credits, u32 dack,
++ int nofail)
++{
++ struct sk_buff *skb;
++ struct cpl_rx_data_ack *req;
++
++ skb = (nofail ? alloc_ctrl_skb(c3cn, sizeof(*req))
++ : alloc_skb(sizeof(*req), GFP_ATOMIC));
++ if (!skb)
++ return 0;
++
++ req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
++ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, c3cn->tid));
++ req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
++ skb->priority = CPL_PRIORITY_ACK;
++ cxgb3_ofld_send(c3cn->cdev, skb);
++ return credits;
++}
++
++static void mk_act_open_req(struct s3_conn *c3cn, struct sk_buff *skb,
++ unsigned int atid, const struct l2t_entry *e)
++{
++ struct cpl_act_open_req *req;
++
++ c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn, atid);
++
++ skb->priority = CPL_PRIORITY_SETUP;
++ req = (struct cpl_act_open_req *)__skb_put(skb, sizeof(*req));
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
++ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
++ req->local_port = c3cn->saddr.sin_port;
++ req->peer_port = c3cn->daddr.sin_port;
++ req->local_ip = c3cn->saddr.sin_addr.s_addr;
++ req->peer_ip = c3cn->daddr.sin_addr.s_addr;
++ req->opt0h = htonl(calc_opt0h(c3cn) | V_L2T_IDX(e->idx) |
++ V_TX_CHANNEL(e->smt_idx));
++ req->opt0l = htonl(calc_opt0l(c3cn));
++ req->params = 0;
++}
++
++/*
++ * Definitions and declarations for CPL handler functions.
++ * =======================================================
++ */
++
++/*
++ * Similar to process_cpl_msg() but takes an extra connection reference around
++ * the call to the handler. Should be used if the handler may drop a
++ * connection reference.
++ */
++static inline void process_cpl_msg_ref(void (*fn) (struct s3_conn *,
++ struct sk_buff *),
++ struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ c3cn_hold(c3cn);
++ process_cpl_msg(fn, c3cn, skb);
++ c3cn_put(c3cn);
++}
++
++/*
++ * Return whether a failed active open has allocated a TID
++ */
++static inline int act_open_has_tid(int status)
++{
++ return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST &&
++ status != CPL_ERR_ARP_MISS;
++}
++
++/*
++ * Returns true if a connection cannot accept new Rx data.
++ */
++static inline int c3cn_no_receive(const struct s3_conn *c3cn)
++{
++ return c3cn->shutdown & C3CN_RCV_SHUTDOWN;
++}
++
++/*
++ * A helper function that aborts a connection and increments the given MIB
++ * counter. The supplied skb is used to generate the ABORT_REQ message if
++ * possible. Must be called with softirqs disabled.
++ */
++static inline void abort_conn(struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ struct sk_buff *abort_skb;
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ abort_skb = __get_cpl_reply_skb(skb, sizeof(struct cpl_abort_req),
++ GFP_ATOMIC);
++ if (abort_skb)
++ s3_send_reset(c3cn, CPL_ABORT_SEND_RST, abort_skb);
++}
++
++/*
++ * Returns whether an ABORT_REQ_RSS message is a negative advice.
++ */
++static inline int is_neg_adv_abort(unsigned int status)
++{
++ return status == CPL_ERR_RTX_NEG_ADVICE ||
++ status == CPL_ERR_PERSIST_NEG_ADVICE;
++}
++
++/*
++ * CPL handler functions.
++ * ======================
++ */
++
++/*
++ * Process a CPL_ACT_ESTABLISH message.
++ */
++static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
++ void *ctx)
++{
++ struct cpl_act_establish *req = cplhdr(skb);
++ unsigned int tid = GET_TID(req);
++ unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
++ struct s3_conn *c3cn = ctx;
++ struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
++
++ c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
++ tid, c3cn, c3cn->state, c3cn->flags);
++ /*
++ * It's OK if the TID is currently in use, the owning connection may
++ * have backlogged its last CPL message(s). Just take it away.
++ */
++ c3cn->tid = tid;
++ c3cn_insert_tid(cdata, c3cn, tid);
++ free_atid(cdev, atid);
++
++ c3cn->qset = G_QNUM(ntohl(skb->csum));
++
++ process_cpl_msg(c3cn_act_establish, c3cn, skb);
++ return 0;
++}
++
++/*
++ * Process an ACT_OPEN_RPL CPL message.
++ */
++static int do_act_open_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
++{
++ struct s3_conn *c3cn = ctx;
++ struct cpl_act_open_rpl *rpl = cplhdr(skb);
++
++ c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
++ rpl->status, c3cn, c3cn->state, c3cn->flags);
++
++ if (act_open_has_tid(rpl->status))
++ cxgb3_queue_tid_release(cdev, GET_TID(rpl));
++
++ process_cpl_msg_ref(active_open_failed, c3cn, skb);
++ return 0;
++}
++
++/*
++ * Handler RX_ISCSI_HDR CPL messages.
++ */
++static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
++{
++ struct s3_conn *c3cn = ctx;
++ process_cpl_msg(process_rx_iscsi_hdr, c3cn, skb);
++ return 0;
++}
++
++/*
++ * Handler for TX_DATA_ACK CPL messages.
++ */
++static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
++{
++ struct s3_conn *c3cn = ctx;
++
++ process_cpl_msg(wr_ack, c3cn, skb);
++ return 0;
++}
++
++/*
++ * Handler for PEER_CLOSE CPL messages.
++ */
++static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
++{
++ struct s3_conn *c3cn = ctx;
++
++ c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++ process_cpl_msg_ref(do_peer_fin, c3cn, skb);
++ return 0;
++}
++
++/*
++ * Handle an ABORT_REQ_RSS CPL message.
++ */
++static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
++{
++ const struct cpl_abort_req_rss *req = cplhdr(skb);
++ struct s3_conn *c3cn = ctx;
++
++ c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ if (is_neg_adv_abort(req->status)) {
++ __kfree_skb(skb);
++ return 0;
++ }
++
++ process_cpl_msg_ref(process_abort_req, c3cn, skb);
++ return 0;
++}
++
++/*
++ * Handle an ABORT_RPL_RSS CPL message.
++ */
++static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
++{
++ struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
++ struct s3_conn *c3cn = ctx;
++
++ c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
++ rpl->status, c3cn, c3cn ? c3cn->state : 0,
++ c3cn ? c3cn->flags : 0UL);
++
++ /*
++ * Ignore replies to post-close aborts indicating that the abort was
++ * requested too late. These connections are terminated when we get
++ * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
++ * arrives the TID is either no longer used or it has been recycled.
++ */
++ if (rpl->status == CPL_ERR_ABORT_FAILED)
++ goto discard;
++
++ /*
++ * Sometimes we've already closed the connection, e.g., a post-close
++ * abort races with ABORT_REQ_RSS, the latter frees the connection
++ * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
++ * but FW turns the ABORT_REQ into a regular one and so we get
++ * ABORT_RPL_RSS with status 0 and no connection. Only on T3A.
++ */
++ if (!c3cn)
++ goto discard;
++
++ process_cpl_msg_ref(process_abort_rpl, c3cn, skb);
++ return 0;
++
++discard:
++ __kfree_skb(skb);
++ return 0;
++}
++
++/*
++ * Handler for CLOSE_CON_RPL CPL messages.
++ */
++static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
++ void *ctx)
++{
++ struct s3_conn *c3cn = ctx;
++
++ c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ process_cpl_msg_ref(process_close_con_rpl, c3cn, skb);
++ return 0;
++}
++
++/*
++ * Definitions and declarations for CPL message processing.
++ * ========================================================
++ */
++
++static void make_established(struct s3_conn *, u32, unsigned int);
++static void act_open_retry_timer(unsigned long);
++static void mk_act_open_req(struct s3_conn *, struct sk_buff *,
++ unsigned int, const struct l2t_entry *);
++static int act_open_rpl_status_to_errno(int);
++static void handle_excess_rx(struct s3_conn *, struct sk_buff *);
++static int abort_status_to_errno(struct s3_conn *, int, int *);
++static void send_abort_rpl(struct sk_buff *, struct t3cdev *, int);
++static struct sk_buff *get_cpl_reply_skb(struct sk_buff *, size_t, gfp_t);
++
++/*
++ * Dequeue and return the first unacknowledged's WR on a connections's pending
++ * list.
++ */
++static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
++{
++ struct sk_buff *skb = c3cn->wr_pending_head;
++
++ if (likely(skb)) {
++ /* Don't bother clearing the tail */
++ c3cn->wr_pending_head = (struct sk_buff *)skb->sp;
++ skb->sp = NULL;
++ }
++ return skb;
++}
++
++/*
++ * Return the first pending WR without removing it from the list.
++ */
++static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
++{
++ return c3cn->wr_pending_head;
++}
++
++static inline void free_wr_skb(struct sk_buff *skb)
++{
++ kfree_skb(skb);
++}
++
++static void purge_wr_queue(struct s3_conn *c3cn)
++{
++ struct sk_buff *skb;
++ while ((skb = dequeue_wr(c3cn)) != NULL)
++ free_wr_skb(skb);
++}
++
++static inline void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
++ int cmd)
++{
++ struct cpl_abort_rpl *rpl = cplhdr(skb);
++
++ rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
++ rpl->wr.wr_lo = htonl(V_WR_TID(tid));
++ OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
++ rpl->cmd = cmd;
++}
++
++/*
++ * CPL message processing ...
++ * ==========================
++ */
++
++/*
++ * Updates connection state from an active establish CPL message. Runs with
++ * the connection lock held.
++ */
++static void c3cn_act_establish(struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ struct cpl_act_establish *req = cplhdr(skb);
++ u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ if (unlikely(c3cn->state != C3CN_STATE_SYN_SENT))
++ printk(KERN_ERR "TID %u expected SYN_SENT, found %d\n",
++ c3cn->tid, c3cn->state);
++
++ c3cn->copied_seq = c3cn->rcv_wup = c3cn->rcv_nxt = rcv_isn;
++ make_established(c3cn, ntohl(req->snd_isn), ntohs(req->tcp_opt));
++
++ if (unlikely(c3cn_flag(c3cn, C3CN_CLOSE_NEEDED))) {
++ /* upper layer has requested closing */
++ abort_conn(c3cn, skb);
++ return;
++ }
++
++ __kfree_skb(skb);
++ if (s3_push_frames(c3cn, 1))
++ cxgb3i_conn_tx_open(c3cn);
++}
++
++/*
++ * Handle active open failures.
++ */
++static void active_open_failed(struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ struct cpl_act_open_rpl *rpl = cplhdr(skb);
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ if (rpl->status == CPL_ERR_CONN_EXIST &&
++ c3cn->retry_timer.function != act_open_retry_timer) {
++ c3cn->retry_timer.function = act_open_retry_timer;
++ c3cn_reset_timer(c3cn, &c3cn->retry_timer,
++ jiffies + HZ / 2);
++ } else
++ fail_act_open(c3cn, act_open_rpl_status_to_errno(rpl->status));
++ __kfree_skb(skb);
++}
++
++/*
++ * Process received pdu for a connection.
++ */
++static void process_rx_iscsi_hdr(struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
++ struct cpl_iscsi_hdr_norss data_cpl;
++ struct cpl_rx_data_ddp_norss ddp_cpl;
++ unsigned int hdr_len, data_len, status;
++ unsigned int len;
++ int err;
++
++ if (unlikely(c3cn_no_receive(c3cn))) {
++ handle_excess_rx(c3cn, skb);
++ return;
++ }
++
++ CXGB3_SKB_CB(skb)->seq = ntohl(hdr_cpl->seq);
++ CXGB3_SKB_CB(skb)->flags = 0;
++
++ skb_reset_transport_header(skb);
++ __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
++
++ len = hdr_len = ntohs(hdr_cpl->len);
++ /* msg coalesce is off or not enough data received */
++ if (skb->len <= hdr_len) {
++ printk(KERN_ERR "%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
++ c3cn->cdev->name, c3cn->tid, skb->len, hdr_len);
++ goto abort_conn;
++ }
++
++ err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
++ sizeof(ddp_cpl));
++ if (err < 0)
++ goto abort_conn;
++
++ skb_ulp_mode(skb) = ULP2_FLAG_DATA_READY;
++ skb_ulp_pdulen(skb) = ntohs(ddp_cpl.len);
++ skb_ulp_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
++ status = ntohl(ddp_cpl.ddp_status);
++
++ if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
++ skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
++ if (status & (1 << RX_DDP_STATUS_DCRC_SHIFT))
++ skb_ulp_mode(skb) |= ULP2_FLAG_DCRC_ERROR;
++ if (status & (1 << RX_DDP_STATUS_PAD_SHIFT))
++ skb_ulp_mode(skb) |= ULP2_FLAG_PAD_ERROR;
++
++ if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
++ err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
++ if (err < 0)
++ goto abort_conn;
++ data_len = ntohs(data_cpl.len);
++ len += sizeof(data_cpl) + data_len;
++ } else if (status & (1 << RX_DDP_STATUS_DDP_SHIFT))
++ skb_ulp_mode(skb) |= ULP2_FLAG_DATA_DDPED;
++
++ c3cn->rcv_nxt = ntohl(ddp_cpl.seq) + skb_ulp_pdulen(skb);
++ __pskb_trim(skb, len);
++ __skb_queue_tail(&c3cn->receive_queue, skb);
++ cxgb3i_conn_pdu_ready(c3cn);
++
++ return;
++
++abort_conn:
++ s3_send_reset(c3cn, CPL_ABORT_SEND_RST, NULL);
++ __kfree_skb(skb);
++}
++
++/*
++ * Process an acknowledgment of WR completion. Advance snd_una and send the
++ * next batch of work requests from the write queue.
++ */
++static void wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
++{
++ struct cpl_wr_ack *hdr = cplhdr(skb);
++ unsigned int credits = ntohs(hdr->credits);
++ u32 snd_una = ntohl(hdr->snd_una);
++
++ c3cn->wr_avail += credits;
++ if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
++ c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
++
++ while (credits) {
++ struct sk_buff *p = peek_wr(c3cn);
++
++ if (unlikely(!p)) {
++ printk(KERN_ERR "%u WR_ACK credits for TID %u with "
++ "nothing pending, state %u\n",
++ credits, c3cn->tid, c3cn->state);
++ break;
++ }
++ if (unlikely(credits < p->csum)) {
++ p->csum -= credits;
++ break;
++ } else {
++ dequeue_wr(c3cn);
++ credits -= p->csum;
++ free_wr_skb(p);
++ }
++ }
++
++ if (unlikely(before(snd_una, c3cn->snd_una)))
++ goto out_free;
++
++ if (c3cn->snd_una != snd_una) {
++ c3cn->snd_una = snd_una;
++ dst_confirm(c3cn->dst_cache);
++ if (c3cn->snd_una == c3cn->snd_nxt)
++ c3cn_reset_flag(c3cn, C3CN_TX_WAIT_IDLE);
++ }
++
++ if (skb_queue_len(&c3cn->write_queue) && s3_push_frames(c3cn, 0))
++ cxgb3i_conn_tx_open(c3cn);
++out_free:
++ __kfree_skb(skb);
++}
++
++/*
++ * Handle a peer FIN.
++ */
++static void do_peer_fin(struct s3_conn *c3cn, struct sk_buff *skb)
++{
++ int keep = 0;
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
++ goto out;
++
++ c3cn->shutdown |= C3CN_RCV_SHUTDOWN;
++ c3cn_set_flag(c3cn, C3CN_DONE);
++
++ switch (c3cn->state) {
++ case C3CN_STATE_ESTABLISHED:
++ break;
++ case C3CN_STATE_CLOSING:
++ c3cn_done(c3cn);
++ break;
++ default:
++ printk(KERN_ERR
++ "%s: TID %u received PEER_CLOSE in bad state %d\n",
++ c3cn->cdev->name, c3cn->tid, c3cn->state);
++ }
++
++ cxgb3i_conn_closing(c3cn);
++out:
++ if (!keep)
++ __kfree_skb(skb);
++}
++
++/*
++ * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
++ * request except that we need to reply to it.
++ */
++static void process_abort_req(struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ int rst_status = CPL_ABORT_NO_RST;
++ const struct cpl_abort_req_rss *req = cplhdr(skb);
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ if (!c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD)) {
++ c3cn_set_flag(c3cn, C3CN_ABORT_REQ_RCVD);
++ c3cn_set_flag(c3cn, C3CN_ABORT_SHUTDOWN);
++ __kfree_skb(skb);
++ return;
++ }
++ c3cn_reset_flag(c3cn, C3CN_ABORT_REQ_RCVD);
++
++ /*
++ * Three cases to consider:
++ * a) We haven't sent an abort_req; close the connection.
++ * b) We have sent a post-close abort_req that will get to TP too late
++ * and will generate a CPL_ERR_ABORT_FAILED reply. The reply will
++ * be ignored and the connection should be closed now.
++ * c) We have sent a regular abort_req that will get to TP too late.
++ * That will generate an abort_rpl with status 0, wait for it.
++ */
++ send_abort_rpl(skb, c3cn->cdev, rst_status);
++
++ if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
++ c3cn->err =
++ abort_status_to_errno(c3cn, req->status, &rst_status);
++
++ c3cn_done(c3cn);
++ }
++}
++
++/*
++ * Process abort replies. We only process these messages if we anticipate
++ * them as the coordination between SW and HW in this area is somewhat lacking
++ * and sometimes we get ABORT_RPLs after we are done with the connection that
++ * originated the ABORT_REQ.
++ */
++static void process_abort_rpl(struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
++ if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_RCVD))
++ c3cn_set_flag(c3cn, C3CN_ABORT_RPL_RCVD);
++ else {
++ c3cn_reset_flag(c3cn, C3CN_ABORT_RPL_RCVD);
++ c3cn_reset_flag(c3cn, C3CN_ABORT_RPL_PENDING);
++ BUG_ON(c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD));
++ c3cn_done(c3cn);
++ }
++ }
++ __kfree_skb(skb);
++}
++
++/*
++ * Process a peer ACK to our FIN.
++ */
++static void process_close_con_rpl(struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ struct cpl_close_con_rpl *rpl = cplhdr(skb);
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ c3cn->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
++
++ if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
++ goto out;
++
++ if (c3cn->state == C3CN_STATE_CLOSING) {
++ c3cn_done(c3cn);
++ } else
++ printk(KERN_ERR
++ "%s: TID %u received CLOSE_CON_RPL in bad state %d\n",
++ c3cn->cdev->name, c3cn->tid, c3cn->state);
++out:
++ kfree_skb(skb);
++}
++
++/*
++ * Random utility functions for CPL message processing ...
++ * =======================================================
++ */
++
++/**
++ * find_best_mtu - find the entry in the MTU table closest to an MTU
++ * @d: TOM state
++ * @mtu: the target MTU
++ *
++ * Returns the index of the value in the MTU table that is closest to but
++ * does not exceed the target MTU.
++ */
++static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
++{
++ int i = 0;
++
++ while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
++ ++i;
++ return i;
++}
++
++static unsigned int select_mss(struct s3_conn *c3cn, unsigned int pmtu)
++{
++ unsigned int idx;
++ struct dst_entry *dst = c3cn->dst_cache;
++ struct t3cdev *cdev = c3cn->cdev;
++ const struct t3c_data *td = T3C_DATA(cdev);
++ u16 advmss = dst_metric(dst, RTAX_ADVMSS);
++
++ if (advmss > pmtu - 40)
++ advmss = pmtu - 40;
++ if (advmss < td->mtus[0] - 40)
++ advmss = td->mtus[0] - 40;
++ idx = find_best_mtu(td, advmss + 40);
++ return idx;
++}
++
++static void fail_act_open(struct s3_conn *c3cn, int errno)
++{
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
++ c3cn, c3cn->state, c3cn->flags);
++
++ c3cn->err = errno;
++ c3cn_done(c3cn);
++}
++
++/*
++ * Assign offload parameters to some connection fields.
++ */
++static void init_offload_conn(struct s3_conn *c3cn,
++ struct t3cdev *cdev,
++ struct dst_entry *dst)
++{
++ BUG_ON(c3cn->cdev != cdev);
++ c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs;
++ c3cn->wr_unacked = 0;
++ c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
++
++ c3cn->ctrl_skb_cache = alloc_skb(CTRL_SKB_LEN, gfp_any());
++ reset_wr_list(c3cn);
++}
++
++static void act_open_retry_timer(unsigned long data)
++{
++ struct sk_buff *skb;
++ struct s3_conn *c3cn = (struct s3_conn *)data;
++
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
++
++ spin_lock(&c3cn->lock);
++ skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_ATOMIC);
++ if (!skb)
++ fail_act_open(c3cn, ENOMEM);
++ else {
++ skb->sk = (struct sock *)c3cn;
++ set_arp_failure_handler(skb, act_open_req_arp_failure);
++ mk_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
++ l2t_send(c3cn->cdev, skb, c3cn->l2t);
++ }
++ spin_unlock(&c3cn->lock);
++ c3cn_put(c3cn);
++}
++
++/*
++ * Convert an ACT_OPEN_RPL status to a Linux errno.
++ */
++static int act_open_rpl_status_to_errno(int status)
++{
++ switch (status) {
++ case CPL_ERR_CONN_RESET:
++ return ECONNREFUSED;
++ case CPL_ERR_ARP_MISS:
++ return EHOSTUNREACH;
++ case CPL_ERR_CONN_TIMEDOUT:
++ return ETIMEDOUT;
++ case CPL_ERR_TCAM_FULL:
++ return ENOMEM;
++ case CPL_ERR_CONN_EXIST:
++ printk(KERN_ERR "ACTIVE_OPEN_RPL: 4-tuple in use\n");
++ return EADDRINUSE;
++ default:
++ return EIO;
++ }
++}
++
++/*
++ * Convert the status code of an ABORT_REQ into a Linux error code. Also
++ * indicate whether RST should be sent in response.
++ */
++static int abort_status_to_errno(struct s3_conn *c3cn,
++ int abort_reason, int *need_rst)
++{
++ switch (abort_reason) {
++ case CPL_ERR_BAD_SYN: /* fall through */
++ case CPL_ERR_CONN_RESET:
++ return c3cn->state == C3CN_STATE_CLOSING ? EPIPE : ECONNRESET;
++ case CPL_ERR_XMIT_TIMEDOUT:
++ case CPL_ERR_PERSIST_TIMEDOUT:
++ case CPL_ERR_FINWAIT2_TIMEDOUT:
++ case CPL_ERR_KEEPALIVE_TIMEDOUT:
++ return ETIMEDOUT;
++ default:
++ return EIO;
++ }
++}
++
++static void send_abort_rpl(struct sk_buff *skb, struct t3cdev *cdev,
++ int rst_status)
++{
++ struct sk_buff *reply_skb;
++ struct cpl_abort_req_rss *req = cplhdr(skb);
++
++ reply_skb = get_cpl_reply_skb(skb, sizeof(struct cpl_abort_rpl),
++ gfp_any());
++
++ reply_skb->priority = CPL_PRIORITY_DATA;
++ set_abort_rpl_wr(reply_skb, GET_TID(req), rst_status);
++ kfree_skb(skb);
++ cxgb3_ofld_send(cdev, reply_skb);
++}
++
++/*
++ * Returns an sk_buff for a reply CPL message of size len. If the input
++ * sk_buff has no other users it is trimmed and reused, otherwise a new buffer
++ * is allocated. The input skb must be of size at least len. Note that this
++ * operation does not destroy the original skb data even if it decides to reuse
++ * the buffer.
++ */
++static struct sk_buff *get_cpl_reply_skb(struct sk_buff *skb, size_t len,
++ gfp_t gfp)
++{
++ if (likely(!skb_cloned(skb))) {
++ BUG_ON(skb->len < len);
++ __skb_trim(skb, len);
++ skb_get(skb);
++ } else {
++ skb = alloc_skb(len, gfp);
++ if (skb)
++ __skb_put(skb, len);
++ }
++ return skb;
++}
++
++/*
++ * Release resources held by an offload connection (TID, L2T entry, etc.)
++ */
++static void t3_release_offload_resources(struct s3_conn *c3cn)
++{
++ struct t3cdev *cdev = c3cn->cdev;
++ unsigned int tid = c3cn->tid;
++
++ if (!cdev)
++ return;
++
++ c3cn->qset = 0;
++
++ kfree_skb(c3cn->ctrl_skb_cache);
++ c3cn->ctrl_skb_cache = NULL;
++
++ if (c3cn->wr_avail != c3cn->wr_max) {
++ purge_wr_queue(c3cn);
++ reset_wr_list(c3cn);
++ }
++
++ if (c3cn->l2t) {
++ l2t_release(L2DATA(cdev), c3cn->l2t);
++ c3cn->l2t = NULL;
++ }
++
++ if (c3cn->state == C3CN_STATE_SYN_SENT) /* we have ATID */
++ free_atid(cdev, tid);
++ else { /* we have TID */
++ cxgb3_remove_tid(cdev, (void *)c3cn, tid);
++ c3cn_put(c3cn);
++ }
++
++ c3cn->cdev = NULL;
++}
++
++/*
++ * Handles Rx data that arrives in a state where the connection isn't
++ * accepting new data.
++ */
++static void handle_excess_rx(struct s3_conn *c3cn, struct sk_buff *skb)
++{
++ if (!c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN))
++ abort_conn(c3cn, skb);
++
++ kfree_skb(skb);
++}
++
++/*
++ * Like get_cpl_reply_skb() but the returned buffer starts out empty.
++ */
++static struct sk_buff *__get_cpl_reply_skb(struct sk_buff *skb, size_t len,
++ gfp_t gfp)
++{
++ if (likely(!skb_cloned(skb) && !skb->data_len)) {
++ __skb_trim(skb, 0);
++ skb_get(skb);
++ } else
++ skb = alloc_skb(len, gfp);
++ return skb;
++}
++
++/*
++ * Completes some final bits of initialization for just established connections
++ * and changes their state to C3CN_STATE_ESTABLISHED.
++ *
++ * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
++ */
++static void make_established(struct s3_conn *c3cn, u32 snd_isn,
++ unsigned int opt)
++{
++ c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
++
++ c3cn->write_seq = c3cn->snd_nxt = c3cn->snd_una = snd_isn;
++
++ /*
++ * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
++ * pass through opt0.
++ */
++ if (cxgb3_rcv_win > (M_RCV_BUFSIZ << 10))
++ c3cn->rcv_wup -= cxgb3_rcv_win - (M_RCV_BUFSIZ << 10);
++
++ dst_confirm(c3cn->dst_cache);
++
++ smp_mb();
++ c3cn_set_state(c3cn, C3CN_STATE_ESTABLISHED);
++}
+--- /dev/null
++++ b/drivers/scsi/cxgb3i/cxgb3i_offload.h
+@@ -0,0 +1,220 @@
++/*
++ * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
++ *
++ * Written by Dimitris Michailidis (dm@chelsio.com)
++ *
++ * This program is distributed in the hope that it will be useful, but WITHOUT
++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
++ * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
++ * release for licensing terms and conditions.
++ */
++
++#ifndef _CXGB3I_OFFLOAD_H
++#define _CXGB3I_OFFLOAD_H
++
++#include <linux/skbuff.h>
++#include <net/tcp.h>
++
++#include "common.h"
++#include "adapter.h"
++#include "t3cdev.h"
++#include "cxgb3_offload.h"
++
++#define cxgb3i_log_error(fmt...) printk(KERN_ERR "cxgb3i: ERR! " fmt)
++#define cxgb3i_log_warn(fmt...) printk(KERN_WARNING "cxgb3i: WARN! " fmt)
++#define cxgb3i_log_info(fmt...) printk(KERN_INFO "cxgb3i: " fmt)
++
++#ifdef __DEBUG_CXGB3I__
++#define cxgb3i_log_debug(fmt, args...) \
++ printk(KERN_INFO "cxgb3i: %s - " fmt, __func__ , ## args)
++#else
++#define cxgb3i_log_debug(fmt...)
++#endif
++
++#ifdef __DEBUG_C3CN_CONN__
++#define c3cn_conn_debug cxgb3i_log_debug
++#else
++#define c3cn_conn_debug(fmt...)
++#endif
++
++/*
++ * Data structure to keep track of cxgb3 connection.
++ */
++struct s3_conn {
++ struct net_device *dev; /* net device of with connection */
++ struct t3cdev *cdev; /* adapter t3cdev for net device */
++ unsigned long flags; /* see c3cn_flags below */
++ int tid; /* ID of TCP Control Block */
++ int qset; /* queue Set used by connection */
++ int mss_idx; /* Maximum Segment Size table index */
++ struct l2t_entry *l2t; /* ARP resolution for offload packets */
++ int wr_max; /* maximum in-flight writes */
++ int wr_avail; /* number of writes available */
++ int wr_unacked; /* writes since last request for */
++ /* completion notification */
++ struct sk_buff *wr_pending_head;/* head of pending write queue */
++ struct sk_buff *wr_pending_tail;/* tail of pending write queue */
++ struct sk_buff *ctrl_skb_cache; /* single entry cached skb for */
++ /* short-term control operations */
++ spinlock_t lock; /* connection status lock */
++ atomic_t refcnt; /* reference count on connection */
++ volatile unsigned int state; /* connection state */
++ struct sockaddr_in saddr; /* source IP/port address */
++ struct sockaddr_in daddr; /* destination IP/port address */
++ struct dst_entry *dst_cache; /* reference to destination route */
++ unsigned char shutdown; /* shutdown status */
++ struct sk_buff_head receive_queue;/* received PDUs */
++ struct sk_buff_head write_queue;/* un-pushed pending writes */
++
++ struct timer_list retry_timer; /* retry timer for various operations */
++ int err; /* connection error status */
++ rwlock_t callback_lock; /* lock for opaque user context */
++ void *user_data; /* opaque user context */
++
++ u32 rcv_nxt; /* what we want to receive next */
++ u32 copied_seq; /* head of yet unread data */
++ u32 rcv_wup; /* rcv_nxt on last window update sent */
++ u32 snd_nxt; /* next sequence we send */
++ u32 snd_una; /* first byte we want an ack for */
++
++ u32 write_seq; /* tail+1 of data held in send buffer */
++};
++
++/* Flags in c3cn->shutdown */
++#define C3CN_RCV_SHUTDOWN 0x1
++#define C3CN_SEND_SHUTDOWN 0x2
++#define C3CN_SHUTDOWN_MASK (C3CN_RCV_SHUTDOWN | C3CN_SEND_SHUTDOWN)
++
++/*
++ * connection state bitmap
++ */
++#define C3CN_STATE_CLOSE 0x1
++#define C3CN_STATE_SYN_SENT 0x2
++#define C3CN_STATE_ESTABLISHED 0x4
++#define C3CN_STATE_CLOSING 0x8
++#define C3CN_STATE_ABORING 0x10
++
++#define C3CN_STATE_MASK 0xFF
++
++static inline unsigned int c3cn_in_state(const struct s3_conn *c3cn,
++ unsigned int states)
++{
++ return states & c3cn->state;
++}
++
++/*
++ * Connection flags -- many to track some close related events.
++ */
++enum c3cn_flags {
++ C3CN_ABORT_RPL_RCVD, /* received one ABORT_RPL_RSS message */
++ C3CN_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
++ C3CN_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
++ C3CN_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
++
++ C3CN_ABORT_RPL_PENDING, /* expecting an abort reply */
++ C3CN_CLOSE_CON_REQUESTED, /* we've sent a close_conn_req */
++ C3CN_TX_DATA_SENT, /* already sent a TX_DATA WR */
++ C3CN_CLOSE_NEEDED, /* need to be closed */
++ C3CN_DONE,
++};
++
++/*
++ * Per adapter data. Linked off of each Ethernet device port on the adapter.
++ * Also available via the t3cdev structure since we have pointers to our port
++ * net_device's there ...
++ */
++struct cxgb3i_sdev_data {
++ struct list_head list; /* links for list of all adapters */
++ struct t3cdev *cdev; /* adapter t3cdev */
++ struct cxgb3_client *client; /* CPL client pointer */
++ struct adap_ports *ports; /* array of adapter ports */
++ unsigned int rx_page_size; /* RX page size */
++ struct sk_buff_head deferq; /* queue for processing replies from */
++ /* worker thread context */
++ struct work_struct deferq_task; /* worker thread */
++};
++#define NDEV2CDATA(ndev) (*(struct cxgb3i_sdev_data **)&(ndev)->ec_ptr)
++#define CXGB3_SDEV_DATA(cdev) NDEV2CDATA((cdev)->lldev)
++
++/*
++ * Primary API routines.
++ */
++void cxgb3i_sdev_cleanup(void);
++int cxgb3i_sdev_init(cxgb3_cpl_handler_func *);
++void cxgb3i_sdev_add(struct t3cdev *, struct cxgb3_client *);
++void cxgb3i_sdev_remove(struct t3cdev *);
++
++struct s3_conn *cxgb3i_c3cn_create(void);
++int cxgb3i_c3cn_connect(struct s3_conn *, struct sockaddr_in *);
++void cxgb3i_c3cn_rx_credits(struct s3_conn *, int);
++int cxgb3i_c3cn_send_pdus(struct s3_conn *, struct sk_buff *, int);
++void cxgb3i_c3cn_release(struct s3_conn *);
++
++/*
++ * Definitions for sk_buff state and ULP mode management.
++ */
++
++struct cxgb3_skb_cb {
++ __u8 flags; /* see C3CB_FLAG_* below */
++ __u8 ulp_mode; /* ULP mode/submode of sk_buff */
++ __u32 seq; /* sequence number */
++ __u32 ddigest; /* ULP rx_data_ddp selected field */
++ __u32 pdulen; /* ULP rx_data_ddp selected field */
++ __u8 ulp_data[16]; /* scratch area for ULP */
++};
++
++#define CXGB3_SKB_CB(skb) ((struct cxgb3_skb_cb *)&((skb)->cb[0]))
++
++#define skb_ulp_mode(skb) (CXGB3_SKB_CB(skb)->ulp_mode)
++#define skb_ulp_ddigest(skb) (CXGB3_SKB_CB(skb)->ddigest)
++#define skb_ulp_pdulen(skb) (CXGB3_SKB_CB(skb)->pdulen)
++#define skb_ulp_data(skb) (CXGB3_SKB_CB(skb)->ulp_data)
++
++enum {
++ C3CB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
++ C3CB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
++ C3CB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */
++ C3CB_FLAG_COMPL = 1 << 4, /* request WR completion */
++};
++
++/*
++ * Top-level CPL message processing used by most CPL messages that
++ * pertain to connections.
++ */
++static inline void process_cpl_msg(void (*fn)(struct s3_conn *,
++ struct sk_buff *),
++ struct s3_conn *c3cn,
++ struct sk_buff *skb)
++{
++ spin_lock(&c3cn->lock);
++ fn(c3cn, skb);
++ spin_unlock(&c3cn->lock);
++}
++
++/*
++ * Opaque version of structure the SGE stores at skb->head of TX_DATA packets
++ * and for which we must reserve space.
++ */
++struct sge_opaque_hdr {
++ void *dev;
++ dma_addr_t addr[MAX_SKB_FRAGS + 1];
++};
++
++/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
++#define TX_HEADER_LEN \
++ (sizeof(struct tx_data_wr) + sizeof(struct sge_opaque_hdr))
++
++void *cxgb3i_alloc_big_mem(unsigned int);
++void cxgb3i_free_big_mem(void *);
++
++/*
++ * get and set private ip for iscsi traffic
++ */
++#define cxgb3i_get_private_ipv4addr(ndev) \
++ (((struct port_info *)(netdev_priv(ndev)))->iscsi_ipv4addr)
++#define cxgb3i_set_private_ipv4addr(ndev, addr) \
++ (((struct port_info *)(netdev_priv(ndev)))->iscsi_ipv4addr) = addr
++
++/* max. connections per adapter */
++#define CXGB3I_MAX_CONN 16384
++#endif /* _CXGB3_OFFLOAD_H */
+--- /dev/null
++++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
+@@ -0,0 +1,741 @@
++/*
++ * cxgb3i_ulp2.c: Chelsio S3xx iSCSI driver.
++ *
++ * Copyright (c) 2008 Chelsio Communications, Inc.
++ *
++ * This program is free software; you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License as published by
++ * the Free Software Foundation.
++ *
++ * Written by: Karen Xie (kxie@chelsio.com)
++ */
++
++#include <linux/skbuff.h>
++#include <scsi/scsi_cmnd.h>
++#include <scsi/scsi_host.h>
++#include <linux/crypto.h>
++
++#include "cxgb3i.h"
++#include "cxgb3i_ulp2.h"
++
++#ifdef __DEBUG_CXGB3I_RX__
++#define cxgb3i_rx_debug cxgb3i_log_debug
++#else
++#define cxgb3i_rx_debug(fmt...)
++#endif
++
++#ifdef __DEBUG_CXGB3I_TX__
++#define cxgb3i_tx_debug cxgb3i_log_debug
++#else
++#define cxgb3i_tx_debug(fmt...)
++#endif
++
++#ifdef __DEBUG_CXGB3I_TAG__
++#define cxgb3i_tag_debug cxgb3i_log_debug
++#else
++#define cxgb3i_tag_debug(fmt...)
++#endif
++
++#ifdef __DEBUG_CXGB3I_DDP__
++#define cxgb3i_ddp_debug cxgb3i_log_debug
++#else
++#define cxgb3i_ddp_debug(fmt...)
++#endif
++
++static struct page *pad_page;
++
++#define ULP2_PGIDX_MAX 4
++#define ULP2_4K_PAGE_SHIFT 12
++#define ULP2_4K_PAGE_MASK (~((1UL << ULP2_4K_PAGE_SHIFT) - 1))
++static unsigned char ddp_page_order[ULP2_PGIDX_MAX];
++static unsigned long ddp_page_size[ULP2_PGIDX_MAX];
++static unsigned char ddp_page_shift[ULP2_PGIDX_MAX];
++static unsigned char sw_tag_idx_bits;
++static unsigned char sw_tag_age_bits;
++
++static void cxgb3i_ddp_page_init(void)
++{
++ int i;
++ unsigned long n = PAGE_SIZE >> ULP2_4K_PAGE_SHIFT;
++
++ if (PAGE_SIZE & (~ULP2_4K_PAGE_MASK)) {
++ cxgb3i_log_debug("PAGE_SIZE 0x%lx is not multiple of 4K, "
++ "ddp disabled.\n", PAGE_SIZE);
++ return;
++ }
++ n = __ilog2_u32(n);
++ for (i = 0; i < ULP2_PGIDX_MAX; i++, n++) {
++ ddp_page_order[i] = n;
++ ddp_page_shift[i] = ULP2_4K_PAGE_SHIFT + n;
++ ddp_page_size[i] = 1 << ddp_page_shift[i];
++ cxgb3i_log_debug("%d, order %u, shift %u, size 0x%lx.\n", i,
++ ddp_page_order[i], ddp_page_shift[i],
++ ddp_page_size[i]);
++ }
++
++ sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
++ sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
++}
++
++static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
++{
++ struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
++
++ req->wr.wr_lo = 0;
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
++ req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
++ V_ULPTX_CMD(ULP_MEM_WRITE));
++ req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) |
++ V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1));
++}
++
++static int set_ddp_map(struct cxgb3i_adapter *snic, struct pagepod_hdr *hdr,
++ unsigned int idx, unsigned int npods,
++ struct scatterlist *sgl, unsigned int sgcnt)
++{
++ struct cxgb3i_ddp_info *ddp = &snic->ddp;
++ struct scatterlist *sg = sgl;
++ unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
++ int i;
++
++ for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
++ struct sk_buff *skb;
++ struct pagepod *ppod;
++ int j, k;
++ skb =
++ alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
++ GFP_ATOMIC);
++ if (!skb) {
++ cxgb3i_log_debug("skb OMM.\n");
++ return -ENOMEM;
++ }
++ skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
++
++ ulp_mem_io_set_hdr(skb, pm_addr);
++ ppod =
++ (struct pagepod *)(skb->head + sizeof(struct ulp_mem_io));
++ memcpy(&(ppod->hdr), hdr, sizeof(struct pagepod));
++ for (j = 0, k = i * 4; j < 5; j++, k++) {
++ if (k < sgcnt) {
++ ppod->addr[j] = cpu_to_be64(sg_dma_address(sg));
++ if (j < 4)
++ sg = sg_next(sg);
++ } else
++ ppod->addr[j] = 0UL;
++ }
++
++ skb->priority = CPL_PRIORITY_CONTROL;
++ cxgb3_ofld_send(snic->tdev, skb);
++ }
++ return 0;
++}
++
++static int clear_ddp_map(struct cxgb3i_adapter *snic, unsigned int idx,
++ unsigned int npods)
++{
++ struct cxgb3i_ddp_info *ddp = &snic->ddp;
++ unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
++ int i;
++
++ for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
++ struct sk_buff *skb;
++ skb =
++ alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
++ GFP_ATOMIC);
++ if (!skb)
++ return -ENOMEM;
++ skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
++ memset((skb->head + sizeof(struct ulp_mem_io)), 0, PPOD_SIZE);
++ ulp_mem_io_set_hdr(skb, pm_addr);
++ skb->priority = CPL_PRIORITY_CONTROL;
++ cxgb3_ofld_send(snic->tdev, skb);
++ }
++ return 0;
++}
++
++static int cxgb3i_ddp_sgl_check(struct scatterlist *sgl, unsigned int sgcnt)
++{
++ struct scatterlist *sg;
++ int i;
++
++ /* make sure the sgl is fit for ddp:
++ * each has the same page size, and
++ * first & last page do not need to be used completely, and
++ * the rest of page must be used completely
++ */
++ for_each_sg(sgl, sg, sgcnt, i) {
++ if ((i && sg->offset) ||
++ ((i != sgcnt - 1) &&
++ (sg->length + sg->offset) != PAGE_SIZE)) {
++ cxgb3i_tag_debug("sg %u/%u, off %u, len %u.\n",
++ i, sgcnt, sg->offset, sg->length);
++ return -EINVAL;
++ }
++ }
++
++ return 0;
++}
++
++static inline int ddp_find_unused_entries(struct cxgb3i_ddp_info *ddp,
++ int start, int max, int count)
++{
++ unsigned int i, j;
++
++ spin_lock(&ddp->map_lock);
++ for (i = start; i <= max;) {
++ for (j = 0; j < count; j++) {
++ if (ddp->map[i + j])
++ break;
++ }
++ if (j == count) {
++ memset(&ddp->map[i], 1, count);
++ spin_unlock(&ddp->map_lock);
++ return i;
++ }
++ i += j + 1;
++ }
++ spin_unlock(&ddp->map_lock);
++ return -EBUSY;
++}
++
++static inline void ddp_unmark_entries(struct cxgb3i_ddp_info *ddp,
++ int start, int count)
++{
++ spin_lock(&ddp->map_lock);
++ memset(&ddp->map[start], 0, count);
++ spin_unlock(&ddp->map_lock);
++}
++
++static inline int sgl_map(struct cxgb3i_adapter *snic,
++ struct scatterlist *sgl, unsigned int sgcnt)
++{
++ struct scatterlist *sg;
++ int i, err;
++
++ for_each_sg(sgl, sg, sgcnt, i) {
++ err = pci_map_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
++ if (err <= 0) {
++ cxgb3i_tag_debug("sgcnt %d/%u, pci map failed %d.\n",
++ i, sgcnt, err);
++ return err;
++ }
++ }
++ return sgcnt;
++}
++
++static inline void sgl_unmap(struct cxgb3i_adapter *snic,
++ struct scatterlist *sgl, unsigned int sgcnt)
++{
++ struct scatterlist *sg;
++ int i;
++
++ for_each_sg(sgl, sg, sgcnt, i) {
++ if (sg_dma_address(sg))
++ pci_unmap_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
++ else
++ break;
++ }
++}
++
++u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *snic, unsigned int tid,
++ u32 sw_tag, unsigned int xferlen,
++ struct scatterlist *sgl, unsigned int sgcnt)
++{
++ struct cxgb3i_ddp_info *ddp = &snic->ddp;
++ struct pagepod_hdr hdr;
++ unsigned int npods;
++ int idx = -1, idx_max;
++ u32 tag;
++ int err;
++
++ if (!ddp || !sgcnt || xferlen < PAGE_SIZE) {
++ cxgb3i_tag_debug("sgcnt %u, xferlen %u < %lu, NO DDP.\n",
++ sgcnt, xferlen, PAGE_SIZE);
++ return RESERVED_ITT;
++ }
++
++ err = cxgb3i_ddp_sgl_check(sgl, sgcnt);
++ if (err < 0) {
++ cxgb3i_tag_debug("sgcnt %u, xferlen %u, SGL check fail.\n",
++ sgcnt, xferlen);
++ return RESERVED_ITT;
++ }
++
++ npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
++ idx_max = ddp->nppods - npods + 1;
++
++ if (ddp->idx_last == ddp->nppods)
++ idx = ddp_find_unused_entries(ddp, 0, idx_max, npods);
++ else {
++ idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1, idx_max,
++ npods);
++ if ((idx < 0) && (ddp->idx_last >= npods))
++ idx = ddp_find_unused_entries(ddp, 0,
++ ddp->idx_last - npods + 1,
++ npods);
++ }
++ if (idx < 0) {
++ cxgb3i_tag_debug("sgcnt %u, xferlen %u, npods %u NO DDP.\n",
++ sgcnt, xferlen, npods);
++ return RESERVED_ITT;
++ }
++
++ err = sgl_map(snic, sgl, sgcnt);
++ if (err < sgcnt)
++ goto unmap_sgl;
++
++ tag = sw_tag | (idx << snic->tag_format.rsvd_shift);
++
++ hdr.rsvd = 0;
++ hdr.vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid));
++ hdr.pgsz_tag_clr = htonl(tag & snic->tag_format.rsvd_tag_mask);
++ hdr.maxoffset = htonl(xferlen);
++ hdr.pgoffset = htonl(sgl->offset);
++
++ if (set_ddp_map(snic, &hdr, idx, npods, sgl, sgcnt) < 0)
++ goto unmap_sgl;
++
++ ddp->idx_last = idx;
++ cxgb3i_tag_debug("tid 0x%x, xfer %u, 0x%x -> ddp 0x%x (0x%x, %u).\n",
++ tid, xferlen, sw_tag, tag, idx, npods);
++ return tag;
++
++unmap_sgl:
++ sgl_unmap(snic, sgl, sgcnt);
++ ddp_unmark_entries(ddp, idx, npods);
++ return RESERVED_ITT;
++}
++
++void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *snic, u32 tag,
++ struct scatterlist *sgl, unsigned int sgcnt)
++{
++ u32 idx = (tag >> snic->tag_format.rsvd_shift) &
++ snic->tag_format.rsvd_mask;
++ unsigned int npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
++
++ if (idx < snic->tag_format.rsvd_mask) {
++ cxgb3i_tag_debug("ddp tag 0x%x, release idx 0x%x, npods %u.\n",
++ tag, idx, npods);
++ clear_ddp_map(snic, idx, npods);
++ ddp_unmark_entries(&snic->ddp, idx, npods);
++ sgl_unmap(snic, sgl, sgcnt);
++ }
++}
++
++int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *cconn, int hcrc, int dcrc)
++{
++ struct iscsi_tcp_conn *tcp_conn = cconn->conn->dd_data;
++ struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
++ struct sk_buff *skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
++ GFP_KERNEL | __GFP_NOFAIL);
++ struct cpl_set_tcb_field *req;
++ u32 submode = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
++
++ /* set up ulp submode and page size */
++ req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
++ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
++ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, c3cn->tid));
++ req->reply = V_NO_REPLY(1);
++ req->cpu_idx = 0;
++ req->word = htons(31);
++ req->mask = cpu_to_be64(0xFF000000);
++ /* the connection page size is always the same as ddp-pgsz0 */
++ req->val = cpu_to_be64(submode << 24);
++ skb->priority = CPL_PRIORITY_CONTROL;
++
++ cxgb3_ofld_send(c3cn->cdev, skb);
++ return 0;
++}
++
++static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn,
++ struct sk_buff *skb)
++{
++ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
++ struct iscsi_segment *segment = &tcp_conn->in.segment;
++ struct iscsi_hdr *hdr = (struct iscsi_hdr *)tcp_conn->in.hdr_buf;
++ unsigned char *buf = (unsigned char *)hdr;
++ unsigned int offset = sizeof(struct iscsi_hdr);
++ int err;
++
++ cxgb3i_rx_debug("conn 0x%p, skb 0x%p, len %u, flag 0x%x.\n",
++ conn, skb, skb->len, skb_ulp_mode(skb));
++
++ /* read bhs */
++ err = skb_copy_bits(skb, 0, buf, sizeof(struct iscsi_hdr));
++ if (err < 0)
++ return err;
++ segment->copied = sizeof(struct iscsi_hdr);
++ /* read ahs */
++ if (hdr->hlength) {
++ unsigned int ahslen = hdr->hlength << 2;
++ /* Make sure we don't overflow */
++ if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf))
++ return -ISCSI_ERR_AHSLEN;
++ err = skb_copy_bits(skb, offset, buf + offset, ahslen);
++ if (err < 0)
++ return err;
++ offset += ahslen;
++ }
++ /* header digest */
++ if (conn->hdrdgst_en)
++ offset += ISCSI_DIGEST_SIZE;
++
++ /* check header digest */
++ segment->status = (conn->hdrdgst_en &&
++ (skb_ulp_mode(skb) & ULP2_FLAG_HCRC_ERROR)) ?
++ ISCSI_SEGMENT_DGST_ERR : 0;
++
++ hdr->itt = ntohl(hdr->itt);
++ segment->total_copied = segment->total_size;
++ tcp_conn->in.hdr = hdr;
++ err = iscsi_tcp_hdr_dissect(conn, hdr);
++ if (err)
++ return err;
++
++ if (tcp_conn->in.datalen) {
++ segment = &tcp_conn->in.segment;
++ segment->status = (conn->datadgst_en &&
++ (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) ?
++ ISCSI_SEGMENT_DGST_ERR : 0;
++ if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
++ cxgb3i_ddp_debug("opcode 0x%x, data %u, ddp'ed.\n",
++ hdr->opcode & ISCSI_OPCODE_MASK,
++ tcp_conn->in.datalen);
++ segment->total_copied = segment->total_size;
++ } else
++ offset += sizeof(struct cpl_iscsi_hdr_norss);
++
++ while (segment->total_copied < segment->total_size) {
++ iscsi_tcp_segment_map(segment, 1);
++ err = skb_copy_bits(skb, offset, segment->data,
++ segment->size);
++ iscsi_tcp_segment_unmap(segment);
++ if (err)
++ return err;
++ segment->total_copied += segment->size;
++ offset += segment->size;
++
++ if (segment->total_copied < segment->total_size)
++ iscsi_tcp_segment_init_sg(segment,
++ sg_next(segment->sg),
++ 0);
++ }
++ err = segment->done(tcp_conn, segment);
++ }
++ return err;
++}
++
++static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
++{
++ u8 submode = 0;
++
++ if (hcrc)
++ submode |= 1;
++ if (dcrc)
++ submode |= 2;
++ skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) | submode;
++}
++
++int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *conn)
++{
++ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
++ struct iscsi_segment *hdr_seg = &tcp_conn->out.segment;
++ struct iscsi_segment *data_seg = &tcp_conn->out.data_segment;
++ unsigned int hdrlen = hdr_seg->total_size;
++ unsigned int datalen = data_seg->total_size;
++ unsigned int padlen = iscsi_padding(datalen);
++ unsigned int copymax = SKB_MAX_HEAD(TX_HEADER_LEN);
++ unsigned int copylen;
++ struct sk_buff *skb;
++ unsigned char *dst;
++ int err = -EAGAIN;
++
++ if (data_seg->data && ((datalen + padlen) < copymax))
++ copylen = hdrlen + datalen + padlen;
++ else
++ copylen = hdrlen;
++
++ /* supports max. 16K pdus, so one skb is enough to hold all the data */
++ skb = alloc_skb(TX_HEADER_LEN + copylen, GFP_ATOMIC);
++ if (!skb)
++ return -EAGAIN;
++
++ skb_reserve(skb, TX_HEADER_LEN);
++ skb_put(skb, copylen);
++ dst = skb->data;
++
++ tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
++
++ memcpy(dst, hdr_seg->data, hdrlen);
++ dst += hdrlen;
++
++ if (!datalen)
++ goto send_pdu;
++
++ if (data_seg->data) {
++ /* data is in a linear buffer */
++ if (copylen > hdrlen) {
++ /* data fits in the skb's headroom */
++ memcpy(dst, data_seg->data, datalen);
++ dst += datalen;
++ if (padlen)
++ memset(dst, 0, padlen);
++ } else {
++ unsigned int offset = 0;
++ while (datalen) {
++ struct page *page = alloc_page(GFP_ATOMIC);
++ int idx = skb_shinfo(skb)->nr_frags;
++ skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
++
++ if (!page)
++ goto free_skb;
++
++ frag->page = page;
++ frag->page_offset = 0;
++ if (datalen > PAGE_SIZE)
++ frag->size = PAGE_SIZE;
++ else
++ frag->size = datalen;
++ memcpy(page_address(page),
++ data_seg->data + offset, frag->size);
++
++ skb_shinfo(skb)->nr_frags++;
++ datalen -= frag->size;
++ offset += frag->size;
++ }
++ }
++ } else {
++ struct scatterlist *sg = data_seg->sg;
++ unsigned int offset = data_seg->sg_offset;
++ while (datalen) {
++ int idx = skb_shinfo(skb)->nr_frags;
++ skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
++ struct page *pg = sg_page(sg);
++
++ get_page(pg);
++ frag->page = pg;
++ frag->page_offset = offset + sg->offset;
++ frag->size = min(sg->length, datalen);
++
++ offset = 0;
++ skb_shinfo(skb)->nr_frags++;
++ datalen -= frag->size;
++ sg = sg_next(sg);
++ }
++ }
++
++ if (skb_shinfo(skb)->nr_frags) {
++ if (padlen) {
++ int idx = skb_shinfo(skb)->nr_frags;
++ skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
++ frag->page = pad_page;
++ frag->page_offset = 0;
++ frag->size = padlen;
++ skb_shinfo(skb)->nr_frags++;
++ }
++ datalen = data_seg->total_size + padlen;
++ skb->data_len += datalen;
++ skb->truesize += datalen;
++ skb->len += datalen;
++ }
++
++send_pdu:
++ err = cxgb3i_c3cn_send_pdus((struct s3_conn *)tcp_conn->sock,
++ skb, MSG_DONTWAIT | MSG_NOSIGNAL);
++ if (err > 0) {
++ int pdulen = hdrlen + datalen + padlen;
++ if (conn->hdrdgst_en)
++ pdulen += ISCSI_DIGEST_SIZE;
++ if (datalen && conn->datadgst_en)
++ pdulen += ISCSI_DIGEST_SIZE;
++
++ hdr_seg->total_copied = hdr_seg->total_size;
++ if (datalen)
++ data_seg->total_copied = data_seg->total_size;
++ conn->txdata_octets += pdulen;
++ return pdulen;
++ }
++
++free_skb:
++ kfree_skb(skb);
++ if (err < 0 && err != -EAGAIN) {
++ cxgb3i_log_error("conn 0x%p, xmit err %d.\n", conn, err);
++ iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
++ return err;
++ }
++ return -EAGAIN;
++}
++
++int cxgb3i_ulp2_init(void)
++{
++ pad_page = alloc_page(GFP_KERNEL);
++ if (!pad_page)
++ return -ENOMEM;
++ memset(page_address(pad_page), 0, PAGE_SIZE);
++ cxgb3i_ddp_page_init();
++ return 0;
++}
++
++void cxgb3i_ulp2_cleanup(void)
++{
++ if (pad_page) {
++ __free_page(pad_page);
++ pad_page = NULL;
++ }
++}
++
++void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn)
++{
++ struct sk_buff *skb;
++ unsigned int read = 0;
++ struct iscsi_conn *conn = c3cn->user_data;
++ int err = 0;
++
++ cxgb3i_rx_debug("cn 0x%p.\n", c3cn);
++
++ read_lock(&c3cn->callback_lock);
++ if (unlikely(!conn || conn->suspend_rx)) {
++ cxgb3i_rx_debug("conn 0x%p, id %d, suspend_rx %lu!\n",
++ conn, conn ? conn->id : 0xFF,
++ conn ? conn->suspend_rx : 0xFF);
++ read_unlock(&c3cn->callback_lock);
++ return;
++ }
++ skb = skb_peek(&c3cn->receive_queue);
++ while (!err && skb) {
++ __skb_unlink(skb, &c3cn->receive_queue);
++ read += skb_ulp_pdulen(skb);
++ err = cxgb3i_conn_read_pdu_skb(conn, skb);
++ __kfree_skb(skb);
++ skb = skb_peek(&c3cn->receive_queue);
++ }
++ read_unlock(&c3cn->callback_lock);
++ if (c3cn) {
++ c3cn->copied_seq += read;
++ cxgb3i_c3cn_rx_credits(c3cn, read);
++ }
++ conn->rxdata_octets += read;
++
++ if (err) {
++ cxgb3i_log_info("conn 0x%p rx failed err %d.\n", conn, err);
++ iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
++ }
++}
++
++void cxgb3i_conn_tx_open(struct s3_conn *c3cn)
++{
++ struct iscsi_conn *conn = c3cn->user_data;
++ struct iscsi_tcp_conn *tcp_conn;
++
++ cxgb3i_tx_debug("cn 0x%p.\n", c3cn);
++ if (conn) {
++ cxgb3i_tx_debug("cn 0x%p, cid %d.\n", c3cn, conn->id);
++ tcp_conn = conn->dd_data;
++ scsi_queue_work(conn->session->host, &conn->xmitwork);
++ }
++}
++
++void cxgb3i_conn_closing(struct s3_conn *c3cn)
++{
++ struct iscsi_conn *conn;
++
++ read_lock(&c3cn->callback_lock);
++ conn = c3cn->user_data;
++ if (conn && c3cn->state != C3CN_STATE_ESTABLISHED)
++ iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
++ read_unlock(&c3cn->callback_lock);
++}
++
++int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
++{
++ struct t3cdev *tdev = snic->tdev;
++ struct cxgb3i_ddp_info *ddp = &snic->ddp;
++ struct ulp_iscsi_info uinfo;
++ unsigned int ppmax, bits, max_bits;
++ int i, err;
++
++ spin_lock_init(&ddp->map_lock);
++
++ err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
++ if (err < 0) {
++ cxgb3i_log_error("%s, failed to get iscsi param err=%d.\n",
++ tdev->name, err);
++ return err;
++ }
++
++ ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
++ max_bits = min(PPOD_IDX_MAX_SIZE,
++ (32 - sw_tag_idx_bits - sw_tag_age_bits));
++ bits = __ilog2_u32(ppmax) + 1;
++ if (bits > max_bits)
++ bits = max_bits;
++ ppmax = (1 << bits) - 1;
++
++ snic->tx_max_size = min_t(unsigned int,
++ uinfo.max_txsz, ULP2_MAX_PKT_SIZE);
++ snic->rx_max_size = min_t(unsigned int,
++ uinfo.max_rxsz, ULP2_MAX_PKT_SIZE);
++
++ snic->tag_format.idx_bits = sw_tag_idx_bits;
++ snic->tag_format.age_bits = sw_tag_age_bits;
++ snic->tag_format.rsvd_bits = bits;
++ snic->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
++ snic->tag_format.rsvd_mask = (1 << snic->tag_format.rsvd_bits) - 1;
++ snic->tag_format.rsvd_tag_mask =
++ (1 << (snic->tag_format.rsvd_bits + PPOD_IDX_SHIFT)) - 1;
++
++ ddp->map = cxgb3i_alloc_big_mem(ppmax);
++ if (!ddp->map) {
++ cxgb3i_log_warn("snic unable to alloc ddp ppod 0x%u, "
++ "ddp disabled.\n", ppmax);
++ return 0;
++ }
++ ddp->llimit = uinfo.llimit;
++ ddp->ulimit = uinfo.ulimit;
++
++ uinfo.tagmask =
++ snic->tag_format.rsvd_mask << snic->tag_format.rsvd_shift;
++ for (i = 0; i < ULP2_PGIDX_MAX; i++)
++ uinfo.pgsz_factor[i] = ddp_page_order[i];
++
++ uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
++
++ err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
++ if (err < 0) {
++ cxgb3i_log_warn("snic unable to set iscsi param err=%d, "
++ "ddp disabled.\n", err);
++ goto free_ppod_map;
++ }
++
++ ddp->nppods = ppmax;
++ ddp->idx_last = ppmax;
++
++ tdev->ulp_iscsi = ddp;
++
++ cxgb3i_log_info("snic nppods %u (0x%x ~ 0x%x), rsvd shift %u, "
++ "bits %u, mask 0x%x, 0x%x, pkt %u,%u.\n",
++ ppmax, ddp->llimit, ddp->ulimit,
++ snic->tag_format.rsvd_shift,
++ snic->tag_format.rsvd_bits,
++ snic->tag_format.rsvd_mask, uinfo.tagmask,
++ snic->tx_max_size, snic->rx_max_size);
++
++ return 0;
++
++free_ppod_map:
++ cxgb3i_free_big_mem(ddp->map);
++ return 0;
++}
++
++void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *snic)
++{
++ u8 *map = snic->ddp.map;
++
++ if (map) {
++ snic->tdev->ulp_iscsi = NULL;
++ spin_lock(&snic->lock);
++ snic->ddp.map = NULL;
++ spin_unlock(&snic->lock);
++ cxgb3i_free_big_mem(map);
++ }
++}
+--- /dev/null
++++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
+@@ -0,0 +1,108 @@
++/*
++ * cxgb3i_ulp2.h: Chelsio S3xx iSCSI driver.
++ *
++ * Copyright (c) 2008 Chelsio Communications, Inc.
++ *
++ * This program is free software; you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License as published by
++ * the Free Software Foundation.
++ *
++ * Written by: Karen Xie (kxie@chelsio.com)
++ */
++
++#ifndef __CXGB3I_ULP2_H__
++#define __CXGB3I_ULP2_H__
++
++#define ULP2_PDU_PAYLOAD_DFLT (16224 - ISCSI_PDU_HEADER_MAX)
++#define PPOD_PAGES_MAX 4
++#define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */
++
++struct pagepod_hdr {
++ u32 vld_tid;
++ u32 pgsz_tag_clr;
++ u32 maxoffset;
++ u32 pgoffset;
++ u64 rsvd;
++};
++
++struct pagepod {
++ struct pagepod_hdr hdr;
++ u64 addr[PPOD_PAGES_MAX + 1];
++};
++
++#define PPOD_SIZE sizeof(struct pagepod) /* 64 */
++#define PPOD_SIZE_SHIFT 6
++
++#define PPOD_COLOR_SHIFT 0
++#define PPOD_COLOR_SIZE 6
++#define PPOD_COLOR_MASK ((1 << PPOD_COLOR_SIZE) - 1)
++
++#define PPOD_IDX_SHIFT PPOD_COLOR_SIZE
++#define PPOD_IDX_MAX_SIZE 24
++
++#define S_PPOD_TID 0
++#define M_PPOD_TID 0xFFFFFF
++#define V_PPOD_TID(x) ((x) << S_PPOD_TID)
++
++#define S_PPOD_VALID 24
++#define V_PPOD_VALID(x) ((x) << S_PPOD_VALID)
++#define F_PPOD_VALID V_PPOD_VALID(1U)
++
++#define S_PPOD_COLOR 0
++#define M_PPOD_COLOR 0x3F
++#define V_PPOD_COLOR(x) ((x) << S_PPOD_COLOR)
++
++#define S_PPOD_TAG 6
++#define M_PPOD_TAG 0xFFFFFF
++#define V_PPOD_TAG(x) ((x) << S_PPOD_TAG)
++
++#define S_PPOD_PGSZ 30
++#define M_PPOD_PGSZ 0x3
++#define V_PPOD_PGSZ(x) ((x) << S_PPOD_PGSZ)
++
++struct cpl_iscsi_hdr_norss {
++ union opcode_tid ot;
++ u16 pdu_len_ddp;
++ u16 len;
++ u32 seq;
++ u16 urg;
++ u8 rsvd;
++ u8 status;
++};
++
++struct cpl_rx_data_ddp_norss {
++ union opcode_tid ot;
++ u16 urg;
++ u16 len;
++ u32 seq;
++ u32 nxt_seq;
++ u32 ulp_crc;
++ u32 ddp_status;
++};
++
++#define RX_DDP_STATUS_IPP_SHIFT 27 /* invalid pagepod */
++#define RX_DDP_STATUS_TID_SHIFT 26 /* tid mismatch */
++#define RX_DDP_STATUS_COLOR_SHIFT 25 /* color mismatch */
++#define RX_DDP_STATUS_OFFSET_SHIFT 24 /* offset mismatch */
++#define RX_DDP_STATUS_ULIMIT_SHIFT 23 /* ulimit error */
++#define RX_DDP_STATUS_TAG_SHIFT 22 /* tag mismatch */
++#define RX_DDP_STATUS_DCRC_SHIFT 21 /* dcrc error */
++#define RX_DDP_STATUS_HCRC_SHIFT 20 /* hcrc error */
++#define RX_DDP_STATUS_PAD_SHIFT 19 /* pad error */
++#define RX_DDP_STATUS_PPP_SHIFT 18 /* pagepod parity error */
++#define RX_DDP_STATUS_LLIMIT_SHIFT 17 /* llimit error */
++#define RX_DDP_STATUS_DDP_SHIFT 16 /* ddp'able */
++#define RX_DDP_STATUS_PMM_SHIFT 15 /* pagepod mismatch */
++
++#define ULP2_FLAG_DATA_READY 0x1
++#define ULP2_FLAG_DATA_DDPED 0x2
++#define ULP2_FLAG_HCRC_ERROR 0x10
++#define ULP2_FLAG_DCRC_ERROR 0x20
++#define ULP2_FLAG_PAD_ERROR 0x40
++
++#define ULP2_MAX_PKT_SIZE 16224
++
++void cxgb3i_conn_closing(struct s3_conn *);
++void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn);
++void cxgb3i_conn_tx_open(struct s3_conn *c3cn);
++#endif
+--- /dev/null
++++ b/drivers/scsi/cxgb3i/Kconfig
+@@ -0,0 +1,7 @@
++config SCSI_CXGB3_ISCSI
++ tristate "Chelsio S3xx iSCSI support"
++ select CHELSIO_T3
++ select SCSI_ISCSI_ATTRS
++ select ISCSI_TCP
++ ---help---
++ This driver supports iSCSI offload for the Chelsio S3 series devices.
+--- /dev/null
++++ b/drivers/scsi/cxgb3i/Makefile
+@@ -0,0 +1,5 @@
++EXTRA_CFLAGS += -I$(TOPDIR)/drivers/net/cxgb3
++
++cxgb3i-y := cxgb3i_init.o cxgb3i_iscsi.o cxgb3i_ulp2.o cxgb3i_offload.o
++
++obj-$(CONFIG_SCSI_CXGB3_ISCSI) += cxgb3i.o
+--- a/drivers/scsi/Kconfig
++++ b/drivers/scsi/Kconfig
+@@ -352,6 +352,8 @@ config ISCSI_TCP
+
+ http://open-iscsi.org
+
++source "drivers/scsi/cxgb3i/Kconfig"
++
+ config SGIWD93_SCSI
+ tristate "SGI WD93C93 SCSI Driver"
+ depends on SGI_HAS_WD93 && SCSI
+--- a/drivers/scsi/Makefile
++++ b/drivers/scsi/Makefile
+@@ -40,6 +40,7 @@ obj-$(CONFIG_LIBFC) += libfc/
+ obj-$(CONFIG_FCOE) += fcoe/
+ obj-$(CONFIG_ISCSI_TCP) += libiscsi.o iscsi_tcp.o
+ obj-$(CONFIG_INFINIBAND_ISER) += libiscsi.o
++obj-$(CONFIG_SCSI_CXGB3_ISCSI) += iscsi_tcp.o cxgb3i/
+ obj-$(CONFIG_SCSI_A4000T) += 53c700.o a4000t.o
+ obj-$(CONFIG_SCSI_ZORRO7XX) += 53c700.o zorro7xx.o
+ obj-$(CONFIG_A3000_SCSI) += a3000.o wd33c93.o
projects / ipfire-2.x.git / blobdiff