1 Subject: add cxgb3i iscsi driver
2 From: Karen Xie <kxie@chelsio.com>
3 References: FATE#304154,bnc#433500
5 Add Chelsio S3 iscsi initiator driver.
6 This patch implements the cxgb3i iscsi connection offload.
8 Signed-off-by: Karen Xie <kxie@chelsio.com>
9 Signed-off-by: Chandra Seetharaman <sekharan@us.ibm.com>
10 Signed-off-by: Hannes Reinecke <hare@suse.de>
14 drivers/scsi/Kconfig | 2
15 drivers/scsi/Makefile | 1
16 drivers/scsi/cxgb3i/Kconfig | 7
17 drivers/scsi/cxgb3i/Makefile | 5
18 drivers/scsi/cxgb3i/cxgb3i.h | 179 +++
19 drivers/scsi/cxgb3i/cxgb3i_init.c | 109 +
20 drivers/scsi/cxgb3i/cxgb3i_iscsi.c | 854 ++++++++++++++
21 drivers/scsi/cxgb3i/cxgb3i_offload.c | 2021 +++++++++++++++++++++++++++++++++++
22 drivers/scsi/cxgb3i/cxgb3i_offload.h | 220 +++
23 drivers/scsi/cxgb3i/cxgb3i_ulp2.c | 741 ++++++++++++
24 drivers/scsi/cxgb3i/cxgb3i_ulp2.h | 108 +
25 11 files changed, 4247 insertions(+)
28 +++ b/drivers/scsi/cxgb3i/cxgb3i.h
31 + * cxgb3i.h: Chelsio S3xx iSCSI driver.
33 + * Copyright (c) 2008 Chelsio Communications, Inc.
35 + * This program is free software; you can redistribute it and/or modify
36 + * it under the terms of the GNU General Public License as published by
37 + * the Free Software Foundation.
39 + * Written by: Karen Xie (kxie@chelsio.com)
45 +#include <linux/module.h>
46 +#include <linux/moduleparam.h>
47 +#include <linux/errno.h>
48 +#include <linux/types.h>
49 +#include <linux/list.h>
50 +#include <linux/netdevice.h>
51 +#include <linux/scatterlist.h>
57 +#include "cxgb3_ctl_defs.h"
58 +#include "cxgb3_offload.h"
59 +#include "firmware_exports.h"
60 +#include "cxgb3i_offload.h"
62 +#include "../iscsi_tcp.h"
64 +#define CXGB3I_SCSI_QDEPTH_DFLT 128
65 +#define CXGB3I_MAX_TARGET CXGB3I_MAX_CONN
66 +#define CXGB3I_MAX_LUN 512
67 +#define ISCSI_PDU_HEADER_MAX (56 + 256) /* bhs + digests + ahs */
69 +struct cxgb3i_adapter;
71 +struct cxgb3i_endpoint;
74 + * struct cxgb3i_tag_format - cxgb3i ulp tag for steering pdu payload
76 + * @idx_bits: # of bits used to store itt (from iscsi laryer)
77 + * @age_bits: # of bits used to store age (from iscsi laryer)
78 + * @rsvd_bits: # of bits used by h/w
79 + * @rsvd_shift: shift left
80 + * @rsvd_mask: bit mask
81 + * @rsvd_tag_mask: h/w tag bit mask
84 +struct cxgb3i_tag_format {
85 + unsigned char idx_bits;
86 + unsigned char age_bits;
87 + unsigned char rsvd_bits;
88 + unsigned char rsvd_shift;
94 + * struct cxgb3i_ddp_info - cxgb3i direct data placement for pdu payload
96 + * @llimit: lower bound of the page pod memory
97 + * @ulimit: upper bound of the page pod memory
98 + * @nppods: # of page pod entries
99 + * @idx_last: page pod entry last used
100 + * @map_lock: lock to synchonize access to the page pod map
101 + * @map: page pod map
103 +struct cxgb3i_ddp_info {
104 + unsigned int llimit;
105 + unsigned int ulimit;
106 + unsigned int nppods;
107 + unsigned int idx_last;
108 + spinlock_t map_lock;
113 + * struct cxgb3i_hba - cxgb3i iscsi structure (per port)
115 + * @snic: cxgb3i adapter containing this port
116 + * @ndev: pointer to netdev structure
117 + * @shost: pointer to scsi host structure
120 + struct cxgb3i_adapter *snic;
121 + struct net_device *ndev;
122 + struct Scsi_Host *shost;
126 + * struct cxgb3i_adapter - cxgb3i adapter structure (per pci)
128 + * @listhead: list head to link elements
129 + * @lock: lock for this structure
130 + * @tdev: pointer to t3cdev used by cxgb3 driver
131 + * @pdev: pointer to pci dev
132 + * @hba_cnt: # of hbas (the same as # of ports)
133 + * @hba: all the hbas on this adapter
134 + * @tx_max_size: max. tx packet size supported
135 + * @rx_max_size: max. rx packet size supported
136 + * @tag_format: ulp tag format settings
137 + * @ddp: ulp ddp state
139 +struct cxgb3i_adapter {
140 + struct list_head list_head;
142 + struct t3cdev *tdev;
143 + struct pci_dev *pdev;
144 + unsigned char hba_cnt;
145 + struct cxgb3i_hba *hba[MAX_NPORTS];
147 + unsigned int tx_max_size;
148 + unsigned int rx_max_size;
150 + struct cxgb3i_tag_format tag_format;
151 + struct cxgb3i_ddp_info ddp;
155 + * struct cxgb3i_conn - cxgb3i iscsi connection
157 + * @tcp_conn: pointer to iscsi_tcp_conn structure
158 + * @listhead: list head to link elements
159 + * @conn: pointer to iscsi_conn structure
160 + * @hba: pointer to the hba this conn. is going through
162 +struct cxgb3i_conn {
163 + struct iscsi_tcp_conn tcp_conn;
164 + struct list_head list_head;
165 + struct cxgb3i_endpoint *cep;
166 + struct iscsi_conn *conn;
167 + struct cxgb3i_hba *hba;
171 + * struct cxgb3i_endpoint - iscsi tcp endpoint
173 + * @c3cn: the h/w tcp connection representation
174 + * @hba: pointer to the hba this conn. is going through
175 + * @cconn: pointer to the associated cxgb3i iscsi connection
177 +struct cxgb3i_endpoint {
178 + struct s3_conn *c3cn;
179 + struct cxgb3i_hba *hba;
180 + struct cxgb3i_conn *cconn;
184 + * Function Prototypes
186 +int cxgb3i_iscsi_init(void);
187 +void cxgb3i_iscsi_cleanup(void);
189 +struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *);
190 +void cxgb3i_adapter_remove(struct t3cdev *);
191 +int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *);
192 +void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *);
194 +struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *);
195 +struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *,
196 + struct net_device *);
197 +void cxgb3i_hba_host_remove(struct cxgb3i_hba *);
199 +int cxgb3i_ulp2_init(void);
200 +void cxgb3i_ulp2_cleanup(void);
201 +int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *, int, int);
202 +void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32,
203 + struct scatterlist *, unsigned int);
204 +u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *, unsigned int,
205 + u32, unsigned int, struct scatterlist *,
207 +int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *);
210 +++ b/drivers/scsi/cxgb3i/cxgb3i_init.c
212 +/* cxgb3i_init.c: Chelsio S3xx iSCSI driver.
214 + * Copyright (c) 2008 Chelsio Communications, Inc.
216 + * This program is free software; you can redistribute it and/or modify
217 + * it under the terms of the GNU General Public License as published by
218 + * the Free Software Foundation.
220 + * Written by: Karen Xie (kxie@chelsio.com)
225 +#define DRV_MODULE_NAME "cxgb3i"
226 +#define DRV_MODULE_VERSION "1.0.0"
227 +#define DRV_MODULE_RELDATE "Jun. 1, 2008"
229 +static char version[] =
230 + "Chelsio S3xx iSCSI Driver " DRV_MODULE_NAME
231 + " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
233 +MODULE_AUTHOR("Karen Xie <kxie@chelsio.com>");
234 +MODULE_DESCRIPTION("Chelsio S3xx iSCSI Driver");
235 +MODULE_LICENSE("GPL");
236 +MODULE_VERSION(DRV_MODULE_VERSION);
238 +static void open_s3_dev(struct t3cdev *);
239 +static void close_s3_dev(struct t3cdev *);
241 +static cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS];
242 +static struct cxgb3_client t3c_client = {
243 + .name = "iscsi_cxgb3",
244 + .handlers = cxgb3i_cpl_handlers,
245 + .add = open_s3_dev,
246 + .remove = close_s3_dev,
250 + * open_s3_dev - register with cxgb3 LLD
251 + * @t3dev: cxgb3 adapter instance
253 +static void open_s3_dev(struct t3cdev *t3dev)
255 + static int vers_printed;
257 + if (!vers_printed) {
258 + printk(KERN_INFO "%s", version);
262 + cxgb3i_log_debug("open cxgb3 %s.\n", t3dev->name);
263 + cxgb3i_sdev_add(t3dev, &t3c_client);
264 + cxgb3i_adapter_add(t3dev);
268 + * close_s3_dev - de-register with cxgb3 LLD
269 + * @t3dev: cxgb3 adapter instance
271 +static void close_s3_dev(struct t3cdev *t3dev)
273 + cxgb3i_log_debug("close cxgb3 %s.\n", t3dev->name);
274 + cxgb3i_adapter_remove(t3dev);
275 + cxgb3i_sdev_remove(t3dev);
279 + * cxgb3i_init_module - module init entry point
281 + * initialize any driver wide global data structures and register itself
282 + * with the cxgb3 module
284 +static int __init cxgb3i_init_module(void)
288 + err = cxgb3i_sdev_init(cxgb3i_cpl_handlers);
292 + err = cxgb3i_iscsi_init();
296 + err = cxgb3i_ulp2_init();
300 + cxgb3_register_client(&t3c_client);
306 + * cxgb3i_exit_module - module cleanup/exit entry point
308 + * go through the driver hba list and for each hba, release any resource held.
309 + * and unregisters iscsi transport and the cxgb3 module
311 +static void __exit cxgb3i_exit_module(void)
313 + cxgb3_unregister_client(&t3c_client);
314 + cxgb3i_ulp2_cleanup();
315 + cxgb3i_iscsi_cleanup();
316 + cxgb3i_sdev_cleanup();
319 +module_init(cxgb3i_init_module);
320 +module_exit(cxgb3i_exit_module);
322 +++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
324 +/* cxgb3i_iscsi.c: Chelsio S3xx iSCSI driver.
326 + * Copyright (c) 2008 Chelsio Communications, Inc.
328 + * This program is free software; you can redistribute it and/or modify
329 + * it under the terms of the GNU General Public License as published by
330 + * the Free Software Foundation.
332 + * Written by: Karen Xie (kxie@chelsio.com)
335 +#include <linux/inet.h>
336 +#include <linux/crypto.h>
337 +#include <net/tcp.h>
338 +#include <scsi/scsi_cmnd.h>
339 +#include <scsi/scsi_device.h>
340 +#include <scsi/scsi_eh.h>
341 +#include <scsi/scsi_host.h>
342 +#include <scsi/scsi.h>
343 +#include <scsi/iscsi_proto.h>
344 +#include <scsi/libiscsi.h>
345 +#include <scsi/scsi_transport_iscsi.h>
349 +static struct scsi_transport_template *cxgb3i_scsi_transport;
350 +static struct scsi_host_template cxgb3i_host_template;
351 +static struct iscsi_transport cxgb3i_iscsi_transport;
353 +static LIST_HEAD(cxgb3i_snic_list);
354 +static DEFINE_RWLOCK(cxgb3i_snic_rwlock);
357 + * cxgb3i_adapter_add - init a s3 adapter structure and any h/w settings
358 + * @snic: pointer to adapter instance
360 +struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *t3dev)
362 + struct cxgb3i_adapter *snic;
363 + struct adapter *adapter = tdev2adap(t3dev);
366 + snic = kzalloc(sizeof(*snic), GFP_KERNEL);
368 + cxgb3i_log_debug("cxgb3 %s, OOM.\n", t3dev->name);
372 + spin_lock_init(&snic->lock);
373 + snic->tdev = t3dev;
374 + snic->pdev = adapter->pdev;
376 + if (cxgb3i_adapter_ulp_init(snic))
379 + for_each_port(adapter, i) {
380 + snic->hba[i] = cxgb3i_hba_host_add(snic, adapter->port[i]);
384 + snic->hba_cnt = adapter->params.nports;
386 + /* add to the list */
387 + write_lock(&cxgb3i_snic_rwlock);
388 + list_add_tail(&snic->list_head, &cxgb3i_snic_list);
389 + write_unlock(&cxgb3i_snic_rwlock);
394 + cxgb3i_adapter_ulp_cleanup(snic);
401 + * cxgb3i_snic_cleanup - release all the resources held and cleanup h/w settings
402 + * @snic: pointer to adapter instance
404 +void cxgb3i_adapter_remove(struct t3cdev *t3dev)
407 + struct cxgb3i_adapter *snic;
409 + /* remove from the list */
410 + read_lock(&cxgb3i_snic_rwlock);
411 + list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
412 + if (snic->tdev == t3dev) {
413 + list_del(&snic->list_head);
417 + write_unlock(&cxgb3i_snic_rwlock);
420 + for (i = 0; i < snic->hba_cnt; i++) {
421 + if (snic->hba[i]) {
422 + cxgb3i_hba_host_remove(snic->hba[i]);
423 + snic->hba[i] = NULL;
427 + /* release ddp resources */
428 + cxgb3i_adapter_ulp_cleanup(snic);
433 +struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *ndev)
435 + struct cxgb3i_adapter *snic;
438 + read_lock(&cxgb3i_snic_rwlock);
439 + list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
440 + for (i = 0; i < snic->hba_cnt; i++) {
441 + if (snic->hba[i]->ndev == ndev) {
442 + read_unlock(&cxgb3i_snic_rwlock);
443 + return snic->hba[i];
447 + read_unlock(&cxgb3i_snic_rwlock);
451 +struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *snic,
452 + struct net_device *ndev)
454 + struct cxgb3i_hba *hba;
455 + struct Scsi_Host *shost;
458 + shost = iscsi_host_alloc(&cxgb3i_host_template,
459 + sizeof(struct cxgb3i_hba),
460 + CXGB3I_SCSI_QDEPTH_DFLT);
462 + cxgb3i_log_info("iscsi_host_alloc failed.\n");
466 + shost->transportt = cxgb3i_scsi_transport;
467 + shost->max_lun = CXGB3I_MAX_LUN;
468 + shost->max_id = CXGB3I_MAX_TARGET;
469 + shost->max_channel = 0;
470 + shost->max_cmd_len = 16;
472 + hba = iscsi_host_priv(shost);
475 + hba->shost = shost;
477 + pci_dev_get(snic->pdev);
478 + err = iscsi_host_add(shost, &snic->pdev->dev);
480 + cxgb3i_log_info("iscsi_host_add failed.\n");
484 + cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n",
485 + shost, hba, shost->host_no);
490 + pci_dev_put(snic->pdev);
491 + scsi_host_put(shost);
495 +void cxgb3i_hba_host_remove(struct cxgb3i_hba *hba)
497 + cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n",
498 + hba->shost, hba, hba->shost->host_no);
499 + iscsi_host_remove(hba->shost);
500 + pci_dev_put(hba->snic->pdev);
501 + iscsi_host_free(hba->shost);
505 + * cxgb3i_ep_connect - establish TCP connection to target portal
506 + * @dst_addr: target IP address
507 + * @non_blocking: blocking or non-blocking call
509 + * Initiates a TCP/IP connection to the dst_addr
511 +static struct iscsi_endpoint *cxgb3i_ep_connect(struct sockaddr *dst_addr,
514 + struct iscsi_endpoint *ep;
515 + struct cxgb3i_endpoint *cep;
516 + struct cxgb3i_hba *hba;
517 + struct s3_conn *c3cn = NULL;
520 + c3cn = cxgb3i_c3cn_create();
522 + cxgb3i_log_info("ep connect OOM.\n");
527 + err = cxgb3i_c3cn_connect(c3cn, (struct sockaddr_in *)dst_addr);
529 + cxgb3i_log_info("ep connect failed.\n");
532 + hba = cxgb3i_hba_find_by_netdev(c3cn->dst_cache->dev);
535 + cxgb3i_log_info("NOT going through cxgbi device.\n");
538 + if (c3cn_in_state(c3cn, C3CN_STATE_CLOSE)) {
540 + cxgb3i_log_info("ep connect unable to connect.\n");
544 + ep = iscsi_create_endpoint(sizeof(*cep));
547 + cxgb3i_log_info("iscsi alloc ep, OOM.\n");
554 + cxgb3i_log_debug("ep 0x%p, 0x%p, c3cn 0x%p, hba 0x%p.\n",
555 + ep, cep, c3cn, hba);
559 + cxgb3i_log_debug("conn 0x%p failed, release.\n", c3cn);
561 + cxgb3i_c3cn_release(c3cn);
562 + return ERR_PTR(err);
566 + * cxgb3i_ep_poll - polls for TCP connection establishement
567 + * @ep: TCP connection (endpoint) handle
568 + * @timeout_ms: timeout value in milli secs
570 + * polls for TCP connect request to complete
572 +static int cxgb3i_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
574 + struct cxgb3i_endpoint *cep = ep->dd_data;
575 + struct s3_conn *c3cn = cep->c3cn;
577 + if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED))
579 + cxgb3i_log_debug("ep 0x%p, c3cn 0x%p established.\n", ep, c3cn);
584 + * cxgb3i_ep_disconnect - teardown TCP connection
585 + * @ep: TCP connection (endpoint) handle
587 + * teardown TCP connection
589 +static void cxgb3i_ep_disconnect(struct iscsi_endpoint *ep)
591 + struct cxgb3i_endpoint *cep = ep->dd_data;
592 + struct cxgb3i_conn *cconn = cep->cconn;
594 + cxgb3i_log_debug("ep 0x%p, cep 0x%p.\n", ep, cep);
596 + if (cconn && cconn->conn) {
597 + struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
600 + * stop the xmit path so the xmit_segment function is
603 + write_lock_bh(&cep->c3cn->callback_lock);
604 + set_bit(ISCSI_SUSPEND_BIT, &cconn->conn->suspend_rx);
605 + cep->c3cn->user_data = NULL;
607 + tcp_conn->sock = NULL;
608 + write_unlock_bh(&cep->c3cn->callback_lock);
611 + cxgb3i_log_debug("ep 0x%p, cep 0x%p, release c3cn 0x%p.\n",
612 + ep, cep, cep->c3cn);
613 + cxgb3i_c3cn_release(cep->c3cn);
614 + iscsi_destroy_endpoint(ep);
618 + * cxgb3i_session_create - create a new iscsi session
619 + * @cmds_max: max # of commands
620 + * @qdepth: scsi queue depth
621 + * @initial_cmdsn: initial iscsi CMDSN for this session
622 + * @host_no: pointer to return host no
624 + * Creates a new iSCSI session
626 +static struct iscsi_cls_session *
627 +cxgb3i_session_create(struct iscsi_endpoint *ep, u16 cmds_max, u16 qdepth,
628 + u32 initial_cmdsn, u32 *host_no)
630 + struct cxgb3i_endpoint *cep;
631 + struct cxgb3i_hba *hba;
632 + struct Scsi_Host *shost;
633 + struct iscsi_cls_session *cls_session;
634 + struct iscsi_session *session;
638 + cxgb3i_log_error("%s, missing endpoint.\n", __func__);
644 + shost = hba->shost;
645 + cxgb3i_log_debug("ep 0x%p, cep 0x%p, hba 0x%p.\n", ep, cep, hba);
646 + BUG_ON(hba != iscsi_host_priv(shost));
648 + *host_no = shost->host_no;
650 + cls_session = iscsi_session_setup(&cxgb3i_iscsi_transport, shost,
652 + sizeof(struct iscsi_tcp_task),
653 + initial_cmdsn, ISCSI_MAX_TARGET);
656 + session = cls_session->dd_data;
658 + for (i = 0; i < session->cmds_max; i++) {
659 + struct iscsi_task *task = session->cmds[i];
660 + struct iscsi_tcp_task *tcp_task = task->dd_data;
662 + task->hdr = &tcp_task->hdr.cmd_hdr;
663 + task->hdr_max = sizeof(tcp_task->hdr) - ISCSI_DIGEST_SIZE;
666 + if (iscsi_r2tpool_alloc(session))
667 + goto remove_session;
669 + return cls_session;
672 + iscsi_session_teardown(cls_session);
677 + * cxgb3i_session_destroy - destroys iscsi session
678 + * @cls_session: pointer to iscsi cls session
680 + * Destroys an iSCSI session instance and releases its all resources held
682 +static void cxgb3i_session_destroy(struct iscsi_cls_session *cls_session)
684 + cxgb3i_log_debug("sess 0x%p.\n", cls_session);
685 + iscsi_r2tpool_free(cls_session->dd_data);
686 + iscsi_session_teardown(cls_session);
690 + * cxgb3i_conn_create - create iscsi connection instance
691 + * @cls_session: pointer to iscsi cls session
694 + * Creates a new iSCSI connection instance for a given session
696 +static inline void cxgb3i_conn_max_xmit_dlength(struct iscsi_conn *conn)
698 + struct cxgb3i_conn *cconn = conn->dd_data;
700 + if (conn->max_xmit_dlength)
701 + conn->max_xmit_dlength = min_t(unsigned int,
702 + conn->max_xmit_dlength,
703 + cconn->hba->snic->tx_max_size -
704 + ISCSI_PDU_HEADER_MAX);
706 + conn->max_xmit_dlength = cconn->hba->snic->tx_max_size -
707 + ISCSI_PDU_HEADER_MAX;
708 + cxgb3i_log_debug("conn 0x%p, max xmit %u.\n",
709 + conn, conn->max_xmit_dlength);
712 +static inline void cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
714 + struct cxgb3i_conn *cconn = conn->dd_data;
716 + if (conn->max_recv_dlength)
717 + conn->max_recv_dlength = min_t(unsigned int,
718 + conn->max_recv_dlength,
719 + cconn->hba->snic->rx_max_size -
720 + ISCSI_PDU_HEADER_MAX);
722 + conn->max_recv_dlength = cconn->hba->snic->rx_max_size -
723 + ISCSI_PDU_HEADER_MAX;
724 + cxgb3i_log_debug("conn 0x%p, max recv %u.\n",
725 + conn, conn->max_recv_dlength);
728 +static struct iscsi_cls_conn *cxgb3i_conn_create(struct iscsi_cls_session
729 + *cls_session, u32 cid)
731 + struct iscsi_cls_conn *cls_conn;
732 + struct iscsi_conn *conn;
733 + struct cxgb3i_conn *cconn;
735 + cxgb3i_log_debug("sess 0x%p, cid %u.\n", cls_session, cid);
737 + cls_conn = iscsi_conn_setup(cls_session, sizeof(*cconn), cid);
740 + conn = cls_conn->dd_data;
742 + cconn = conn->dd_data;
743 + cconn->tcp_conn.iscsi_conn = conn;
744 + cconn->conn = conn;
750 + * cxgb3i_conn_xmit_segment - transmit segment
751 + * @conn: pointer to iscsi conn
753 +static int cxgb3i_conn_xmit_segment(struct iscsi_conn *conn)
755 + struct cxgb3i_conn *cconn = conn->dd_data;
756 + struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
757 + struct iscsi_segment *segment = &tcp_conn->out.segment;
759 + if (segment->total_copied < segment->total_size)
760 + return cxgb3i_conn_ulp2_xmit(conn);
765 + * cxgb3i_conn_bind - binds iscsi sess, conn and endpoint together
766 + * @cls_session: pointer to iscsi cls session
767 + * @cls_conn: pointer to iscsi cls conn
768 + * @transport_eph: 64-bit EP handle
769 + * @is_leading: leading connection on this session?
771 + * Binds together an iSCSI session, an iSCSI connection and a
772 + * TCP connection. This routine returns error code if the TCP
773 + * connection does not belong on the device iSCSI sess/conn is bound
776 +static int cxgb3i_conn_bind(struct iscsi_cls_session *cls_session,
777 + struct iscsi_cls_conn *cls_conn,
778 + u64 transport_eph, int is_leading)
780 + struct iscsi_conn *conn = cls_conn->dd_data;
781 + struct cxgb3i_conn *cconn = conn->dd_data;
782 + struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
783 + struct iscsi_endpoint *ep;
784 + struct cxgb3i_endpoint *cep;
785 + struct s3_conn *c3cn;
788 + ep = iscsi_lookup_endpoint(transport_eph);
792 + cxgb3i_log_debug("ep 0x%p, cls sess 0x%p, cls conn 0x%p.\n",
793 + ep, cls_session, cls_conn);
795 + err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
802 + read_lock(&c3cn->callback_lock);
803 + /* mnc: TODO don't abuse iscsi_tcp fields */
804 + tcp_conn->sock = (struct socket *)c3cn;
805 + c3cn->user_data = conn;
806 + read_unlock(&c3cn->callback_lock);
808 + cconn->hba = cep->hba;
810 + cep->cconn = cconn;
812 + cxgb3i_conn_max_xmit_dlength(conn);
813 + cxgb3i_conn_max_recv_dlength(conn);
815 + spin_lock_bh(&conn->session->lock);
816 + sprintf(conn->portal_address, NIPQUAD_FMT,
817 + NIPQUAD(c3cn->daddr.sin_addr.s_addr));
818 + conn->portal_port = ntohs(c3cn->daddr.sin_port);
819 + spin_unlock_bh(&conn->session->lock);
821 + tcp_conn->xmit_segment = cxgb3i_conn_xmit_segment;
822 + iscsi_tcp_hdr_recv_prep(tcp_conn);
828 + * cxgb3i_conn_get_param - return iscsi connection parameter to caller
829 + * @cls_conn: pointer to iscsi cls conn
830 + * @param: parameter type identifier
831 + * @buf: buffer pointer
833 + * returns iSCSI connection parameters
835 +static int cxgb3i_conn_get_param(struct iscsi_cls_conn *cls_conn,
836 + enum iscsi_param param, char *buf)
838 + struct iscsi_conn *conn = cls_conn->dd_data;
841 + cxgb3i_log_debug("cls_conn 0x%p, param %d.\n", cls_conn, param);
844 + case ISCSI_PARAM_CONN_PORT:
845 + spin_lock_bh(&conn->session->lock);
846 + len = sprintf(buf, "%hu\n", conn->portal_port);
847 + spin_unlock_bh(&conn->session->lock);
849 + case ISCSI_PARAM_CONN_ADDRESS:
850 + spin_lock_bh(&conn->session->lock);
851 + len = sprintf(buf, "%s\n", conn->portal_address);
852 + spin_unlock_bh(&conn->session->lock);
855 + return iscsi_conn_get_param(cls_conn, param, buf);
861 +static int cxgb3i_conn_set_param(struct iscsi_cls_conn *cls_conn,
862 + enum iscsi_param param, char *buf, int buflen)
864 + struct iscsi_conn *conn = cls_conn->dd_data;
865 + struct iscsi_session *session = conn->session;
866 + struct cxgb3i_conn *cconn = conn->dd_data;
867 + int value, err = 0;
870 + case ISCSI_PARAM_HDRDGST_EN:
871 + err = iscsi_set_param(cls_conn, param, buf, buflen);
872 + if (!err && conn->hdrdgst_en)
873 + cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en,
874 + conn->datadgst_en);
876 + case ISCSI_PARAM_DATADGST_EN:
877 + err = iscsi_set_param(cls_conn, param, buf, buflen);
878 + if (!err && conn->datadgst_en)
879 + cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en,
880 + conn->datadgst_en);
882 + case ISCSI_PARAM_MAX_R2T:
883 + sscanf(buf, "%d", &value);
884 + if (value <= 0 || !is_power_of_2(value))
886 + if (session->max_r2t == value)
888 + iscsi_r2tpool_free(session);
889 + err = iscsi_set_param(cls_conn, param, buf, buflen);
890 + if (!err && iscsi_r2tpool_alloc(session))
892 + case ISCSI_PARAM_MAX_RECV_DLENGTH:
893 + err = iscsi_set_param(cls_conn, param, buf, buflen);
894 + cxgb3i_conn_max_recv_dlength(conn);
896 + case ISCSI_PARAM_MAX_XMIT_DLENGTH:
897 + err = iscsi_set_param(cls_conn, param, buf, buflen);
898 + cxgb3i_conn_max_xmit_dlength(conn);
901 + return iscsi_set_param(cls_conn, param, buf, buflen);
907 + * cxgb3i_host_set_param - configure host (adapter) related parameters
908 + * @shost: scsi host pointer
909 + * @param: parameter type identifier
910 + * @buf: buffer pointer
912 +static int cxgb3i_host_set_param(struct Scsi_Host *shost,
913 + enum iscsi_host_param param,
914 + char *buf, int buflen)
916 + struct cxgb3i_hba *hba = iscsi_host_priv(shost);
918 + cxgb3i_log_debug("param %d, buf %s.\n", param, buf);
920 + if (hba && param == ISCSI_HOST_PARAM_IPADDRESS) {
921 + __be32 addr = in_aton(buf);
922 + cxgb3i_set_private_ipv4addr(hba->ndev, addr);
926 + return iscsi_host_get_param(shost, param, buf);
930 + * cxgb3i_host_get_param - returns host (adapter) related parameters
931 + * @shost: scsi host pointer
932 + * @param: parameter type identifier
933 + * @buf: buffer pointer
935 +static int cxgb3i_host_get_param(struct Scsi_Host *shost,
936 + enum iscsi_host_param param, char *buf)
938 + struct cxgb3i_hba *hba = iscsi_host_priv(shost);
942 + cxgb3i_log_debug("hba %s, param %d.\n", hba->ndev->name, param);
945 + case ISCSI_HOST_PARAM_HWADDRESS:
946 + for (i = 0; i < 6; i++)
948 + sprintf(buf + len, "%02x.",
949 + hba->ndev->dev_addr[i]);
953 + case ISCSI_HOST_PARAM_NETDEV_NAME:
954 + len = sprintf(buf, "%s\n", hba->ndev->name);
956 + case ISCSI_HOST_PARAM_IPADDRESS:
960 + addr = cxgb3i_get_private_ipv4addr(hba->ndev);
961 + len = sprintf(buf, "%u.%u.%u.%u", NIPQUAD(addr));
965 + return iscsi_host_get_param(shost, param, buf);
971 + * cxgb3i_conn_get_stats - returns iSCSI stats
972 + * @cls_conn: pointer to iscsi cls conn
973 + * @stats: pointer to iscsi statistic struct
975 +static void cxgb3i_conn_get_stats(struct iscsi_cls_conn *cls_conn,
976 + struct iscsi_stats *stats)
978 + struct iscsi_conn *conn = cls_conn->dd_data;
980 + stats->txdata_octets = conn->txdata_octets;
981 + stats->rxdata_octets = conn->rxdata_octets;
982 + stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
983 + stats->dataout_pdus = conn->dataout_pdus_cnt;
984 + stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
985 + stats->datain_pdus = conn->datain_pdus_cnt;
986 + stats->r2t_pdus = conn->r2t_pdus_cnt;
987 + stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
988 + stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
989 + stats->digest_err = 0;
990 + stats->timeout_err = 0;
991 + stats->custom_length = 1;
992 + strcpy(stats->custom[0].desc, "eh_abort_cnt");
993 + stats->custom[0].value = conn->eh_abort_cnt;
996 +static inline u32 tag_base(struct cxgb3i_tag_format *format,
997 + unsigned int idx, unsigned int age)
999 + u32 sw_bits = idx | (age << format->idx_bits);
1000 + u32 tag = sw_bits >> format->rsvd_shift;
1002 + tag <<= format->rsvd_bits + format->rsvd_shift;
1003 + tag |= sw_bits & ((1 << format->rsvd_shift) - 1);
1007 +static inline void cxgb3i_parse_tag(struct cxgb3i_tag_format *format,
1008 + u32 tag, u32 *rsvd_bits, u32 *sw_bits)
1011 + *rsvd_bits = (tag >> format->rsvd_shift) & format->rsvd_mask;
1013 + *sw_bits = (tag >> (format->rsvd_shift + format->rsvd_bits))
1014 + << format->rsvd_shift;
1015 + *sw_bits |= tag & ((1 << format->rsvd_shift) - 1);
1020 +static void cxgb3i_parse_itt(struct iscsi_conn *conn, itt_t itt,
1021 + int *idx, int *age)
1023 + struct cxgb3i_conn *cconn = conn->dd_data;
1024 + struct cxgb3i_adapter *snic = cconn->hba->snic;
1027 + cxgb3i_parse_tag(&snic->tag_format, itt, NULL, &sw_bits);
1029 + *idx = sw_bits & ISCSI_ITT_MASK;
1031 + *age = (sw_bits >> snic->tag_format.idx_bits) & ISCSI_AGE_MASK;
1034 +static int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
1036 + struct scsi_cmnd *sc = task->sc;
1037 + struct iscsi_conn *conn = task->conn;
1038 + struct iscsi_session *sess = conn->session;
1039 + struct cxgb3i_conn *cconn = conn->dd_data;
1040 + struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
1041 + struct cxgb3i_adapter *snic = cconn->hba->snic;
1042 + u32 sw_tag = tag_base(&snic->tag_format, task->itt, sess->age);
1043 + u32 tag = RESERVED_ITT;
1045 + if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE)) {
1046 + struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
1048 + cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag,
1049 + scsi_out(sc)->length,
1050 + scsi_out(sc)->table.sgl,
1051 + scsi_out(sc)->table.nents);
1053 + if (tag == RESERVED_ITT)
1054 + tag = sw_tag | (snic->tag_format.rsvd_mask <<
1055 + snic->tag_format.rsvd_shift);
1056 + *hdr_itt = htonl(tag);
1060 +static void cxgb3i_release_itt(struct iscsi_task *task, itt_t hdr_itt)
1062 + struct scsi_cmnd *sc = task->sc;
1063 + struct iscsi_conn *conn = task->conn;
1064 + struct cxgb3i_conn *cconn = conn->dd_data;
1065 + struct cxgb3i_adapter *snic = cconn->hba->snic;
1067 + hdr_itt = ntohl(hdr_itt);
1068 + if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE))
1069 + cxgb3i_ddp_tag_release(snic, hdr_itt,
1070 + scsi_out(sc)->table.sgl,
1071 + scsi_out(sc)->table.nents);
1075 + * cxgb3i_host_template -- Scsi_Host_Template structure
1076 + * used when registering with the scsi mid layer
1078 +static struct scsi_host_template cxgb3i_host_template = {
1079 + .module = THIS_MODULE,
1080 + .name = "Chelsio S3xx iSCSI Initiator",
1081 + .proc_name = "cxgb3i",
1082 + .queuecommand = iscsi_queuecommand,
1083 + .change_queue_depth = iscsi_change_queue_depth,
1084 + .can_queue = 128 * (ISCSI_DEF_XMIT_CMDS_MAX - 1),
1085 + .sg_tablesize = SG_ALL,
1086 + .max_sectors = 0xFFFF,
1087 + .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
1088 + .eh_abort_handler = iscsi_eh_abort,
1089 + .eh_device_reset_handler = iscsi_eh_device_reset,
1090 + .eh_target_reset_handler = iscsi_eh_host_reset,
1091 + .use_clustering = DISABLE_CLUSTERING,
1095 +static struct iscsi_transport cxgb3i_iscsi_transport = {
1096 + .owner = THIS_MODULE,
1098 + .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
1099 + | CAP_DATADGST | CAP_DIGEST_OFFLOAD,
1100 + .param_mask = ISCSI_MAX_RECV_DLENGTH |
1101 + ISCSI_MAX_XMIT_DLENGTH |
1102 + ISCSI_HDRDGST_EN |
1103 + ISCSI_DATADGST_EN |
1104 + ISCSI_INITIAL_R2T_EN |
1106 + ISCSI_IMM_DATA_EN |
1107 + ISCSI_FIRST_BURST |
1109 + ISCSI_PDU_INORDER_EN |
1110 + ISCSI_DATASEQ_INORDER_EN |
1113 + ISCSI_CONN_ADDRESS |
1114 + ISCSI_EXP_STATSN |
1115 + ISCSI_PERSISTENT_PORT |
1116 + ISCSI_PERSISTENT_ADDRESS |
1117 + ISCSI_TARGET_NAME | ISCSI_TPGT |
1118 + ISCSI_USERNAME | ISCSI_PASSWORD |
1119 + ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
1120 + ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
1121 + ISCSI_LU_RESET_TMO |
1122 + ISCSI_PING_TMO | ISCSI_RECV_TMO |
1123 + ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
1124 + .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
1125 + ISCSI_HOST_INITIATOR_NAME | ISCSI_HOST_NETDEV_NAME,
1126 + .get_host_param = cxgb3i_host_get_param,
1127 + .set_host_param = cxgb3i_host_set_param,
1128 + /* session management */
1129 + .create_session = cxgb3i_session_create,
1130 + .destroy_session = cxgb3i_session_destroy,
1131 + .get_session_param = iscsi_session_get_param,
1132 + /* connection management */
1133 + .create_conn = cxgb3i_conn_create,
1134 + .bind_conn = cxgb3i_conn_bind,
1135 + .destroy_conn = iscsi_conn_teardown,
1136 + .start_conn = iscsi_conn_start,
1137 + .stop_conn = iscsi_conn_stop,
1138 + .get_conn_param = cxgb3i_conn_get_param,
1139 + .set_param = cxgb3i_conn_set_param,
1140 + .get_stats = cxgb3i_conn_get_stats,
1141 + /* pdu xmit req. from user space */
1142 + .send_pdu = iscsi_conn_send_pdu,
1144 + .init_task = iscsi_tcp_task_init,
1145 + .xmit_task = iscsi_tcp_task_xmit,
1146 + .cleanup_task = iscsi_tcp_cleanup_task,
1147 + .parse_itt = cxgb3i_parse_itt,
1148 + .reserve_itt = cxgb3i_reserve_itt,
1149 + .release_itt = cxgb3i_release_itt,
1150 + /* TCP connect/disconnect */
1151 + .ep_connect = cxgb3i_ep_connect,
1152 + .ep_poll = cxgb3i_ep_poll,
1153 + .ep_disconnect = cxgb3i_ep_disconnect,
1154 + /* Error recovery timeout call */
1155 + .session_recovery_timedout = iscsi_session_recovery_timedout,
1158 +int cxgb3i_iscsi_init(void)
1160 + cxgb3i_scsi_transport =
1161 + iscsi_register_transport(&cxgb3i_iscsi_transport);
1162 + if (!cxgb3i_scsi_transport) {
1163 + cxgb3i_log_error("Could not register cxgb3i transport.\n");
1166 + cxgb3i_log_debug("cxgb3i transport 0x%p.\n", cxgb3i_scsi_transport);
1170 +void cxgb3i_iscsi_cleanup(void)
1172 + if (cxgb3i_scsi_transport) {
1173 + cxgb3i_log_debug("cxgb3i transport 0x%p.\n",
1174 + cxgb3i_scsi_transport);
1175 + iscsi_unregister_transport(&cxgb3i_iscsi_transport);
1179 +++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
1182 + * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
1184 + * Written by Dimitris Michailidis (dm@chelsio.com)
1186 + * This program is distributed in the hope that it will be useful, but WITHOUT
1187 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1188 + * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
1189 + * release for licensing terms and conditions.
1192 +#include <linux/if_vlan.h>
1193 +#include <linux/version.h>
1195 +#include "cxgb3_defs.h"
1196 +#include "cxgb3_ctl_defs.h"
1197 +#include "firmware_exports.h"
1198 +#include "cxgb3i_offload.h"
1199 +#include "cxgb3i_ulp2.h"
1201 +static int cxgb3_rcv_win = 256 * 1024;
1202 +module_param(cxgb3_rcv_win, int, 0644);
1203 +MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
1205 +static int cxgb3_snd_win = 64 * 1024;
1206 +module_param(cxgb3_snd_win, int, 0644);
1207 +MODULE_PARM_DESC(cxgb3_snd_win, "TCP send window in bytes (default=64KB)");
1209 +static int cxgb3_rx_credit_thres = 10 * 1024;
1210 +module_param(cxgb3_rx_credit_thres, int, 0644);
1211 +MODULE_PARM_DESC(rx_credit_thres,
1212 + "RX credits return threshold in bytes (default=10KB)");
1214 +static unsigned int cxgb3_max_connect = 8 * 1024;
1215 +module_param(cxgb3_max_connect, uint, 0644);
1216 +MODULE_PARM_DESC(cxgb3_max_connect, "Max. # of connections (default=8092)");
1218 +static unsigned int cxgb3_sport_base = 20000;
1219 +module_param(cxgb3_sport_base, uint, 0644);
1220 +MODULE_PARM_DESC(cxgb3_sport_base, "starting port number (default=20000)");
1222 +#ifdef __DEBUG_C3CN_TX__
1223 +#define c3cn_tx_debug cxgb3i_log_debug
1225 +#define c3cn_tx_debug(fmt...)
1228 +/* connection flags */
1229 +static inline void c3cn_set_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
1231 + __set_bit(flag, &c3cn->flags);
1232 + c3cn_conn_debug("c3cn 0x%p, set %d, s 0x%x, f 0x%lx.\n",
1233 + c3cn, flag, c3cn->state, c3cn->flags);
1236 +static inline void c3cn_reset_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
1238 + __clear_bit(flag, &c3cn->flags);
1239 + c3cn_conn_debug("c3cn 0x%p, clear %d, s 0x%x, f 0x%lx.\n",
1240 + c3cn, flag, c3cn->state, c3cn->flags);
1243 +static inline int c3cn_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
1247 + return test_bit(flag, &c3cn->flags);
1250 +/* connection state */
1251 +static void c3cn_set_state(struct s3_conn *c3cn, int state)
1253 + c3cn_conn_debug("c3cn 0x%p state -> 0x%x.\n", c3cn, state);
1254 + c3cn->state = state;
1257 +/* connection reference count */
1258 +static inline void c3cn_hold(struct s3_conn *c3cn)
1260 + atomic_inc(&c3cn->refcnt);
1263 +static inline void c3cn_put(struct s3_conn *c3cn)
1265 + if (atomic_dec_and_test(&c3cn->refcnt)) {
1266 + c3cn_conn_debug("free c3cn 0x%p, 0x%x, 0x%lx.\n",
1267 + c3cn, c3cn->state, c3cn->flags);
1272 +/* minimal port allocation management scheme */
1273 +static spinlock_t sport_map_lock;
1274 +static unsigned int sport_map_next;
1275 +static unsigned long *sport_map;
1278 + * Find a free source port in our allocation map. We use a very simple rotor
1279 + * scheme to look for the next free port.
1281 + * If a source port has been specified make sure that it doesn't collide with
1282 + * our normal source port allocation map. If it's outside the range of our
1283 + * allocation scheme just let them use it.
1285 +static int c3cn_get_port(struct s3_conn *c3cn)
1287 + unsigned int start;
1292 + if (c3cn->saddr.sin_port != 0) {
1293 + int sport = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
1296 + if (sport < 0 || sport >= cxgb3_max_connect)
1298 + spin_lock(&sport_map_lock);
1299 + err = __test_and_set_bit(sport, sport_map);
1300 + spin_unlock(&sport_map_lock);
1301 + return err ? -EADDRINUSE : 0;
1304 + spin_lock(&sport_map_lock);
1305 + start = sport_map_next;
1307 + unsigned int new = sport_map_next;
1308 + if (++sport_map_next >= cxgb3_max_connect)
1309 + sport_map_next = 0;
1310 + if (!(__test_and_set_bit(new, sport_map))) {
1311 + spin_unlock(&sport_map_lock);
1312 + c3cn_conn_debug("reserve port %u.\n",
1313 + cxgb3_sport_base + new);
1314 + c3cn->saddr.sin_port = htons(cxgb3_sport_base + new);
1317 + } while (sport_map_next != start);
1318 + spin_unlock(&sport_map_lock);
1321 + return -EADDRNOTAVAIL;
1325 + * Deallocate a source port from the allocation map. If the source port is
1326 + * outside our allocation range just return -- the caller is responsible for
1327 + * keeping track of their port usage outside of our allocation map.
1329 +static void c3cn_put_port(struct s3_conn *c3cn)
1331 + if (c3cn->saddr.sin_port) {
1332 + int old = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
1333 + c3cn->saddr.sin_port = 0;
1335 + if (old < 0 || old >= cxgb3_max_connect)
1338 + c3cn_conn_debug("release port %u.\n", cxgb3_sport_base + old);
1339 + spin_lock(&sport_map_lock);
1340 + __clear_bit(old, sport_map);
1341 + spin_unlock(&sport_map_lock);
1345 +static void c3cn_reset_timer(struct s3_conn *c3cn, struct timer_list *timer,
1346 + unsigned long expires)
1348 + if (!mod_timer(timer, expires))
1352 +typedef int (cxgb3_cpl_handler_decl) (struct t3cdev *,
1353 + struct sk_buff *, void *);
1355 +static cxgb3_cpl_handler_decl do_act_establish;
1356 +static cxgb3_cpl_handler_decl do_act_open_rpl;
1357 +static cxgb3_cpl_handler_decl do_wr_ack;
1358 +static cxgb3_cpl_handler_decl do_peer_close;
1359 +static cxgb3_cpl_handler_decl do_abort_req;
1360 +static cxgb3_cpl_handler_decl do_abort_rpl;
1361 +static cxgb3_cpl_handler_decl do_close_con_rpl;
1362 +static cxgb3_cpl_handler_decl do_iscsi_hdr;
1364 +static LIST_HEAD(cxgb3_list);
1365 +static DEFINE_MUTEX(cxgb3_list_lock);
1368 + * For ULP connections HW may inserts digest bytes into the pdu. This array
1369 + * contains the compensating extra lengths for ULP packets. It is indexed by
1370 + * a packet's ULP submode.
1372 +static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 };
1375 + * Return the length of any HW additions that will be made to a Tx packet.
1376 + * Such additions can happen for some types of ULP packets.
1378 +static inline unsigned int ulp_extra_len(const struct sk_buff *skb)
1380 + return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3];
1384 + * Size of WRs in bytes. Note that we assume all devices we are handling have
1385 + * the same WR size.
1387 +static unsigned int wrlen __read_mostly;
1390 + * The number of WRs needed for an skb depends on the number of page fragments
1391 + * in the skb and whether it has any payload in its main body. This maps the
1392 + * length of the gather list represented by an skb into the # of necessary WRs.
1394 +static unsigned int skb_wrs[MAX_SKB_FRAGS + 2] __read_mostly;
1396 +static void s3_init_wr_tab(unsigned int wr_len)
1400 + if (skb_wrs[1]) /* already initialized */
1403 + for (i = 1; i < ARRAY_SIZE(skb_wrs); i++) {
1404 + int sgl_len = (3 * i) / 2 + (i & 1);
1407 + skb_wrs[i] = (sgl_len <= wr_len
1408 + ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
1411 + wrlen = wr_len * 8;
1415 + * cxgb3i API operations.
1418 + * large memory chunk allocation/release
1420 +void *cxgb3i_alloc_big_mem(unsigned int size)
1422 + void *p = kmalloc(size, GFP_KERNEL);
1424 + p = vmalloc(size);
1426 + memset(p, 0, size);
1430 +void cxgb3i_free_big_mem(void *addr)
1432 + if (is_vmalloc_addr(addr))
1438 +void cxgb3i_sdev_cleanup(void)
1441 + cxgb3i_free_big_mem(sport_map);
1444 +int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers)
1446 + cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish;
1447 + cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl;
1448 + cpl_handlers[CPL_PEER_CLOSE] = do_peer_close;
1449 + cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req;
1450 + cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl;
1451 + cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl;
1452 + cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack;
1453 + cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr;
1455 + if (cxgb3_max_connect > CXGB3I_MAX_CONN)
1456 + cxgb3_max_connect = CXGB3I_MAX_CONN;
1457 + sport_map = cxgb3i_alloc_big_mem(DIV_ROUND_UP(cxgb3_max_connect,
1459 + sizeof(unsigned long)));
1465 +void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client)
1467 + struct cxgb3i_sdev_data *cdata;
1468 + struct adap_ports *ports;
1469 + struct ofld_page_info rx_page_info;
1470 + unsigned int wr_len;
1473 + cdata = kzalloc(sizeof *cdata, GFP_KERNEL);
1476 + ports = kzalloc(sizeof *ports, GFP_KERNEL);
1479 + cdata->ports = ports;
1481 + if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 ||
1482 + cdev->ctl(cdev, GET_PORTS, cdata->ports) < 0 ||
1483 + cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0)
1486 + s3_init_wr_tab(wr_len);
1488 + INIT_LIST_HEAD(&cdata->list);
1489 + cdata->cdev = cdev;
1490 + cdata->client = client;
1491 + cdata->rx_page_size = rx_page_info.page_size;
1492 + skb_queue_head_init(&cdata->deferq);
1494 + for (i = 0; i < ports->nports; i++)
1495 + NDEV2CDATA(ports->lldevs[i]) = cdata;
1497 + mutex_lock(&cxgb3_list_lock);
1498 + list_add_tail(&cdata->list, &cxgb3_list);
1499 + mutex_unlock(&cxgb3_list_lock);
1508 +void cxgb3i_sdev_remove(struct t3cdev *cdev)
1510 + struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
1511 + struct adap_ports *ports = cdata->ports;
1514 + for (i = 0; i < ports->nports; i++)
1515 + NDEV2CDATA(ports->lldevs[i]) = NULL;
1517 + mutex_lock(&cxgb3_list_lock);
1518 + list_del(&cdata->list);
1519 + mutex_unlock(&cxgb3_list_lock);
1526 + * Return TRUE if the specified net device is for a port on one of our
1527 + * registered adapters.
1529 +static int is_cxgb3_dev(struct net_device *dev)
1531 + struct cxgb3i_sdev_data *cdata;
1533 + mutex_lock(&cxgb3_list_lock);
1534 + list_for_each_entry(cdata, &cxgb3_list, list) {
1535 + struct adap_ports *ports = cdata->ports;
1538 + for (i = 0; i < ports->nports; i++)
1539 + if (dev == ports->lldevs[i]) {
1540 + mutex_unlock(&cxgb3_list_lock);
1544 + mutex_unlock(&cxgb3_list_lock);
1549 + * Primary cxgb3 API operations.
1550 + * =============================
1553 +static int s3_push_frames(struct s3_conn *, int);
1554 +static int s3_send_reset(struct s3_conn *, int, struct sk_buff *);
1555 +static void t3_release_offload_resources(struct s3_conn *);
1556 +static void mk_close_req(struct s3_conn *);
1558 +struct s3_conn *cxgb3i_c3cn_create(void)
1560 + struct s3_conn *c3cn;
1562 + c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL);
1566 + c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn);
1569 + spin_lock_init(&c3cn->lock);
1570 + atomic_set(&c3cn->refcnt, 1);
1571 + skb_queue_head_init(&c3cn->receive_queue);
1572 + skb_queue_head_init(&c3cn->write_queue);
1573 + setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn);
1574 + rwlock_init(&c3cn->callback_lock);
1579 +static inline void s3_purge_write_queue(struct s3_conn *c3cn)
1581 + struct sk_buff *skb;
1583 + while ((skb = __skb_dequeue(&c3cn->write_queue)))
1587 +static void c3cn_done(struct s3_conn *c3cn)
1589 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
1590 + c3cn, c3cn->state, c3cn->flags);
1592 + c3cn_put_port(c3cn);
1593 + t3_release_offload_resources(c3cn);
1594 + c3cn_set_state(c3cn, C3CN_STATE_CLOSE);
1595 + c3cn->shutdown = C3CN_SHUTDOWN_MASK;
1596 + cxgb3i_conn_closing(c3cn);
1599 +static void c3cn_close(struct s3_conn *c3cn)
1601 + int data_lost, old_state;
1603 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
1604 + c3cn, c3cn->state, c3cn->flags);
1606 + dst_confirm(c3cn->dst_cache);
1608 + spin_lock_bh(&c3cn->lock);
1609 + c3cn->shutdown |= C3CN_SHUTDOWN_MASK;
1612 + * We need to flush the receive buffs. We do this only on the
1613 + * descriptor close, not protocol-sourced closes, because the
1614 + * reader process may not have drained the data yet! Make a note
1615 + * of whether any received data will be lost so we can decide whether
1618 + data_lost = skb_queue_len(&c3cn->receive_queue);
1619 + __skb_queue_purge(&c3cn->receive_queue);
1621 + if (c3cn->state == C3CN_STATE_CLOSE)
1622 + /* Nothing if we are already closed */
1623 + c3cn_conn_debug("c3cn 0x%p, 0x%x, already closed.\n",
1624 + c3cn, c3cn->state);
1625 + else if (data_lost || c3cn->state == C3CN_STATE_SYN_SENT) {
1626 + c3cn_conn_debug("c3cn 0x%p, 0x%x -> closing, send reset.\n",
1627 + c3cn, c3cn->state);
1628 + /* Unread data was tossed, zap the connection. */
1629 + s3_send_reset(c3cn, CPL_ABORT_SEND_RST, NULL);
1631 + } else if (c3cn->state == C3CN_STATE_ESTABLISHED) {
1632 + c3cn_conn_debug("c3cn 0x%p, est. -> closing, send close_req.\n",
1634 + c3cn_set_state(c3cn, C3CN_STATE_CLOSING);
1635 + mk_close_req(c3cn);
1639 + old_state = c3cn->state;
1640 + c3cn_hold(c3cn); /* must last past the potential destroy() */
1642 + spin_unlock_bh(&c3cn->lock);
1645 + * There are no more user references at this point. Grab the
1646 + * connection lock and finish the close.
1648 + local_bh_disable();
1649 + spin_lock(&c3cn->lock);
1652 + * Because the connection was orphaned before the spin_lock()
1653 + * either the backlog or a BH may have already destroyed it.
1656 + if (old_state != C3CN_STATE_CLOSE && c3cn->state == C3CN_STATE_CLOSE)
1659 + if (c3cn->state == C3CN_STATE_CLOSE)
1660 + s3_purge_write_queue(c3cn);
1663 + spin_unlock(&c3cn->lock);
1664 + local_bh_enable();
1668 +void cxgb3i_c3cn_release(struct s3_conn *c3cn)
1670 + c3cn_conn_debug("c3cn 0x%p, s 0x%x, f 0x%lx.\n",
1671 + c3cn, c3cn->state, c3cn->flags);
1672 + if (likely(c3cn->state != C3CN_STATE_SYN_SENT))
1675 + c3cn_set_flag(c3cn, C3CN_CLOSE_NEEDED);
1681 + * Local utility routines used to implement primary cxgb3 API operations.
1682 + * ======================================================================
1685 +static u32 s3_send_rx_credits(struct s3_conn *, u32, u32, int);
1686 +static int act_open(struct s3_conn *, struct net_device *);
1687 +static void mk_act_open_req(struct s3_conn *, struct sk_buff *,
1688 + unsigned int, const struct l2t_entry *);
1689 +static void skb_entail(struct s3_conn *, struct sk_buff *, int);
1691 +static inline void reset_wr_list(struct s3_conn *c3cn)
1693 + c3cn->wr_pending_head = NULL;
1697 + * Add a WR to a connections's list of pending WRs. This is a singly-linked
1698 + * list of sk_buffs operating as a FIFO. The head is kept in wr_pending_head
1699 + * and the tail in wr_pending_tail.
1701 +static inline void enqueue_wr(struct s3_conn *c3cn,
1702 + struct sk_buff *skb)
1707 + * We want to take an extra reference since both us and the driver
1708 + * need to free the packet before it's really freed. We know there's
1709 + * just one user currently so we use atomic_set rather than skb_get
1710 + * to avoid the atomic op.
1712 + atomic_set(&skb->users, 2);
1714 + if (!c3cn->wr_pending_head)
1715 + c3cn->wr_pending_head = skb;
1717 + c3cn->wr_pending_tail->sp = (void *)skb;
1718 + c3cn->wr_pending_tail = skb;
1722 + * The next two functions calculate the option 0 value for a connection.
1724 +static inline int compute_wscale(int win)
1727 + while (wscale < 14 && (65535<<wscale) < win)
1732 +static inline unsigned int calc_opt0h(struct s3_conn *c3cn)
1734 + int wscale = compute_wscale(cxgb3_rcv_win);
1735 + return V_KEEP_ALIVE(1) |
1737 + V_WND_SCALE(wscale) |
1738 + V_MSS_IDX(c3cn->mss_idx);
1741 +static inline unsigned int calc_opt0l(struct s3_conn *c3cn)
1743 + return V_ULP_MODE(ULP_MODE_ISCSI) |
1744 + V_RCV_BUFSIZ(cxgb3_rcv_win>>10);
1747 +static inline void make_tx_data_wr(struct s3_conn *c3cn,
1748 + struct sk_buff *skb, int len)
1750 + struct tx_data_wr *req;
1752 + skb_reset_transport_header(skb);
1753 + req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
1754 + req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
1755 + req->wr_lo = htonl(V_WR_TID(c3cn->tid));
1756 + req->sndseq = htonl(c3cn->snd_nxt);
1757 + /* len includes the length of any HW ULP additions */
1758 + req->len = htonl(len);
1759 + req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx));
1760 + /* V_TX_ULP_SUBMODE sets both the mode and submode */
1761 + req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) |
1762 + V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
1764 + if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
1766 + req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
1767 + V_TX_CPU_IDX(c3cn->qset));
1769 + /* Sendbuffer is in units of 32KB.
1771 + req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
1772 + c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
1777 + * cxgb3_egress_dev - return the cxgb3 egress device
1778 + * @root_dev: the root device anchoring the search
1779 + * @c3cn: the connection used to determine egress port in bonding mode
1780 + * @context: in bonding mode, indicates a connection set up or failover
1782 + * Return egress device or NULL if the egress device isn't one of our ports.
1784 + * Given a root network device it returns the physical egress device that is a
1785 + * descendant of the root device. The root device may be either a physical
1786 + * device, in which case it is the device returned, or a virtual device, such
1787 + * as a VLAN or bonding device. In case of a bonding device the search
1788 + * considers the decisions of the bonding device given its mode to locate the
1789 + * correct egress device.
1791 +static struct net_device *cxgb3_egress_dev(struct net_device *root_dev,
1792 + struct s3_conn *c3cn,
1795 + while (root_dev) {
1796 + if (root_dev->priv_flags & IFF_802_1Q_VLAN)
1797 + root_dev = vlan_dev_real_dev(root_dev);
1798 + else if (is_cxgb3_dev(root_dev))
1806 +static struct rtable *find_route(__be32 saddr, __be32 daddr,
1807 + __be16 sport, __be16 dport)
1809 + struct rtable *rt;
1810 + struct flowi fl = {
1817 + .proto = IPPROTO_TCP,
1821 + .dport = dport } } };
1823 + if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
1828 +int cxgb3i_c3cn_connect(struct s3_conn *c3cn, struct sockaddr_in *usin)
1830 + struct rtable *rt;
1831 + struct net_device *dev;
1832 + struct cxgb3i_sdev_data *cdata;
1833 + struct t3cdev *cdev;
1837 + if (usin->sin_family != AF_INET)
1838 + return -EAFNOSUPPORT;
1840 + /* get a source port if one hasn't been provided */
1841 + err = c3cn_get_port(c3cn);
1845 + c3cn_conn_debug("c3cn 0x%p get port %u.\n",
1846 + c3cn, ntohs(c3cn->saddr.sin_port));
1848 + c3cn->daddr.sin_port = usin->sin_port;
1849 + c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr;
1851 + rt = find_route(c3cn->saddr.sin_addr.s_addr,
1852 + c3cn->daddr.sin_addr.s_addr,
1853 + c3cn->saddr.sin_port,
1854 + c3cn->daddr.sin_port);
1856 + c3cn_conn_debug("NO route to 0x%x, port %u.\n",
1857 + c3cn->daddr.sin_addr.s_addr,
1858 + ntohs(c3cn->daddr.sin_port));
1859 + return -ENETUNREACH;
1862 + if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
1863 + c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n",
1864 + c3cn->daddr.sin_addr.s_addr,
1865 + ntohs(c3cn->daddr.sin_port));
1867 + return -ENETUNREACH;
1870 + if (!c3cn->saddr.sin_addr.s_addr)
1871 + c3cn->saddr.sin_addr.s_addr = rt->rt_src;
1873 + /* now commit destination to connection */
1874 + c3cn->dst_cache = &rt->u.dst;
1876 + /* try to establish an offloaded connection */
1877 + dev = cxgb3_egress_dev(c3cn->dst_cache->dev, c3cn, 0);
1878 + if (dev == NULL) {
1879 + c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn);
1880 + return -ENETUNREACH;
1882 + cdata = NDEV2CDATA(dev);
1883 + cdev = cdata->cdev;
1885 + sipv4 = cxgb3i_get_private_ipv4addr(dev);
1887 + c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn);
1888 + sipv4 = c3cn->saddr.sin_addr.s_addr;
1889 + cxgb3i_set_private_ipv4addr(dev, sipv4);
1891 + c3cn->saddr.sin_addr.s_addr = sipv4;
1893 + c3cn_conn_debug("c3cn 0x%p, %u.%u.%u.%u,%u-%u.%u.%u.%u,%u SYN_SENT.\n",
1894 + c3cn, NIPQUAD(c3cn->saddr.sin_addr.s_addr),
1895 + ntohs(c3cn->saddr.sin_port),
1896 + NIPQUAD(c3cn->daddr.sin_addr.s_addr),
1897 + ntohs(c3cn->daddr.sin_port));
1899 + c3cn_set_state(c3cn, C3CN_STATE_SYN_SENT);
1901 + if (!act_open(c3cn, dev))
1905 + * If we get here, we don't have an offload connection so simply
1906 + * return a failure.
1911 + * This trashes the connection and releases the local port,
1914 + c3cn_conn_debug("c3cn 0x%p -> CLOSE.\n", c3cn);
1915 + c3cn_set_state(c3cn, C3CN_STATE_CLOSE);
1917 + c3cn_put_port(c3cn);
1918 + c3cn->daddr.sin_port = 0;
1923 + * Set of states for which we should return RX credits.
1925 +#define CREDIT_RETURN_STATE (C3CN_STATE_ESTABLISHED)
1928 + * Called after some received data has been read. It returns RX credits
1929 + * to the HW for the amount of data processed.
1931 +void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied)
1933 + struct t3cdev *cdev;
1935 + u32 credits, dack = 0;
1937 + if (!c3cn_in_state(c3cn, CREDIT_RETURN_STATE))
1940 + credits = c3cn->copied_seq - c3cn->rcv_wup;
1941 + if (unlikely(!credits))
1944 + cdev = c3cn->cdev;
1946 + if (unlikely(cxgb3_rx_credit_thres == 0))
1949 + dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
1952 + * For coalescing to work effectively ensure the receive window has
1953 + * at least 16KB left.
1955 + must_send = credits + 16384 >= cxgb3_rcv_win;
1957 + if (must_send || credits >= cxgb3_rx_credit_thres)
1958 + c3cn->rcv_wup += s3_send_rx_credits(c3cn, credits, dack,
1963 + * Generic ARP failure handler that discards the buffer.
1965 +static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb)
1971 + * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
1972 + * connection's send queue and sends them on to T3. Must be called with the
1973 + * connection's lock held. Returns the amount of send buffer space that was
1974 + * freed as a result of sending queued data to T3.
1976 +static int s3_push_frames(struct s3_conn *c3cn, int req_completion)
1978 + int total_size = 0;
1979 + struct sk_buff *skb;
1980 + struct t3cdev *cdev;
1981 + struct cxgb3i_sdev_data *cdata;
1983 + if (unlikely(c3cn_in_state(c3cn,
1984 + C3CN_STATE_SYN_SENT | C3CN_STATE_CLOSE)))
1988 + * We shouldn't really be called at all after an abort but check just
1991 + if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN)))
1994 + cdev = c3cn->cdev;
1995 + cdata = CXGB3_SDEV_DATA(cdev);
1997 + while (c3cn->wr_avail
1998 + && (skb = skb_peek(&c3cn->write_queue)) != NULL
1999 + && !c3cn_flag(c3cn, C3CN_TX_WAIT_IDLE)) {
2001 + int len = skb->len; /* length before skb_push */
2002 + int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
2003 + int wrs_needed = skb_wrs[frags];
2005 + if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
2008 + WARN_ON(frags >= ARRAY_SIZE(skb_wrs) || wrs_needed < 1);
2010 + if (c3cn->wr_avail < wrs_needed)
2013 + __skb_unlink(skb, &c3cn->write_queue);
2014 + skb->priority = CPL_PRIORITY_DATA;
2015 + skb->csum = wrs_needed; /* remember this until the WR_ACK */
2016 + c3cn->wr_avail -= wrs_needed;
2017 + c3cn->wr_unacked += wrs_needed;
2018 + enqueue_wr(c3cn, skb);
2020 + if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
2021 + len += ulp_extra_len(skb);
2022 + make_tx_data_wr(c3cn, skb, len);
2023 + c3cn->snd_nxt += len;
2024 + if ((req_completion
2025 + && c3cn->wr_unacked == wrs_needed)
2026 + || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL)
2027 + || c3cn->wr_unacked >= c3cn->wr_max / 2) {
2028 + struct work_request_hdr *wr = cplhdr(skb);
2030 + wr->wr_hi |= htonl(F_WR_COMPL);
2031 + c3cn->wr_unacked = 0;
2033 + CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR;
2034 + } else if (skb->data[0] == FW_WROPCODE_OFLD_CLOSE_CON)
2035 + c3cn_set_flag(c3cn, C3CN_CLOSE_CON_REQUESTED);
2037 + total_size += skb->truesize;
2038 + set_arp_failure_handler(skb, arp_failure_discard);
2039 + l2t_send(cdev, skb, c3cn->l2t);
2041 + return total_size;
2045 + * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant
2046 + * and send it along.
2048 +static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb)
2050 + struct cpl_abort_req *req = cplhdr(skb);
2052 + c3cn_conn_debug("tdev 0x%p.\n", cdev);
2054 + req->cmd = CPL_ABORT_NO_RST;
2055 + cxgb3_ofld_send(cdev, skb);
2059 + * Send an ABORT_REQ message. Cannot fail. This routine makes sure we do
2060 + * not send multiple ABORT_REQs for the same connection and also that we do
2061 + * not try to send a message after the connection has closed. Returns 1 if
2062 + * an ABORT_REQ wasn't generated after all, 0 otherwise.
2064 +static int s3_send_reset(struct s3_conn *c3cn, int mode,
2065 + struct sk_buff *skb)
2067 + struct cpl_abort_req *req;
2068 + unsigned int tid = c3cn->tid;
2070 + if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN) || !c3cn->cdev)) {
2076 + c3cn_conn_debug("c3cn 0x%p, mode %d, flag ABORT_RPL + ABORT_SHUT.\n",
2079 + c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING);
2080 + c3cn_set_flag(c3cn, C3CN_ABORT_SHUTDOWN);
2082 + /* Purge the send queue so we don't send anything after an abort. */
2083 + s3_purge_write_queue(c3cn);
2086 + skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
2087 + skb->priority = CPL_PRIORITY_DATA;
2088 + set_arp_failure_handler(skb, abort_arp_failure);
2090 + req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
2091 + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
2092 + req->wr.wr_lo = htonl(V_WR_TID(tid));
2093 + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
2094 + req->rsvd0 = htonl(c3cn->snd_nxt);
2095 + req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT);
2098 + l2t_send(c3cn->cdev, skb, c3cn->l2t);
2103 + * Add a list of skbs to a connection send queue. This interface is intended
2104 + * for use by in-kernel ULPs. The skbs must comply with the max size limit of
2105 + * the device and have a headroom of at least TX_HEADER_LEN bytes.
2107 +int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb, int flags)
2109 + struct sk_buff *next;
2110 + int err, copied = 0;
2112 + spin_lock_bh(&c3cn->lock);
2114 + if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED)) {
2120 + if (c3cn->err || (c3cn->shutdown & C3CN_SEND_SHUTDOWN))
2124 + if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
2125 + c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
2132 + skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
2133 + copied += skb->len;
2134 + c3cn->write_seq += skb->len + ulp_extra_len(skb);
2138 + if (likely(skb_queue_len(&c3cn->write_queue)))
2139 + s3_push_frames(c3cn, 1);
2140 + spin_unlock_bh(&c3cn->lock);
2144 + if (copied == 0 && err == -EPIPE)
2145 + copied = c3cn->err ? c3cn->err : -EPIPE;
2150 + * Low-level utility routines for primary API functions.
2151 + * =====================================================
2153 +/* routines to implement CPL message processing */
2154 +static void c3cn_act_establish(struct s3_conn *, struct sk_buff *);
2155 +static void active_open_failed(struct s3_conn *, struct sk_buff *);
2156 +static void wr_ack(struct s3_conn *, struct sk_buff *);
2157 +static void do_peer_fin(struct s3_conn *, struct sk_buff *);
2158 +static void process_abort_req(struct s3_conn *, struct sk_buff *);
2159 +static void process_abort_rpl(struct s3_conn *, struct sk_buff *);
2160 +static void process_close_con_rpl(struct s3_conn *, struct sk_buff *);
2161 +static void process_rx_iscsi_hdr(struct s3_conn *, struct sk_buff *);
2163 +static struct sk_buff *__get_cpl_reply_skb(struct sk_buff *, size_t, gfp_t);
2165 +static void fail_act_open(struct s3_conn *, int);
2166 +static void init_offload_conn(struct s3_conn *, struct t3cdev *,
2167 + struct dst_entry *);
2170 + * Insert a connection into the TID table and take an extra reference.
2172 +static inline void c3cn_insert_tid(struct cxgb3i_sdev_data *cdata,
2173 + struct s3_conn *c3cn,
2177 + cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid);
2180 +static inline void free_atid(struct t3cdev *cdev, unsigned int tid)
2182 + struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid);
2188 + * This function is intended for allocations of small control messages.
2189 + * Such messages go as immediate data and usually the pakets are freed
2190 + * immediately. We maintain a cache of one small sk_buff and use it whenever
2191 + * it is available (has a user count of 1). Otherwise we get a fresh buffer.
2193 +#define CTRL_SKB_LEN 120
2195 +static struct sk_buff *alloc_ctrl_skb(const struct s3_conn *c3cn,
2198 + struct sk_buff *skb = c3cn->ctrl_skb_cache;
2200 + if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
2201 + __skb_trim(skb, 0);
2202 + atomic_set(&skb->users, 2);
2203 + } else if (likely(!in_atomic()))
2204 + skb = alloc_skb(len, GFP_ATOMIC | __GFP_NOFAIL);
2206 + skb = alloc_skb(len, GFP_ATOMIC);
2211 + * Handle an ARP failure for an active open.
2213 +static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
2215 + struct s3_conn *c3cn = (struct s3_conn *)skb->sk;
2217 + c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
2220 + spin_lock(&c3cn->lock);
2221 + if (c3cn->state == C3CN_STATE_SYN_SENT) {
2222 + fail_act_open(c3cn, EHOSTUNREACH);
2225 + spin_unlock(&c3cn->lock);
2230 + * Send an active open request.
2232 +static int act_open(struct s3_conn *c3cn, struct net_device *dev)
2234 + struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev);
2235 + struct t3cdev *cdev = cdata->cdev;
2236 + struct dst_entry *dst = c3cn->dst_cache;
2237 + struct sk_buff *skb;
2239 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2240 + c3cn, c3cn->state, c3cn->flags);
2242 + * Initialize connection data. Note that the flags and ULP mode are
2243 + * initialized higher up ...
2246 + c3cn->cdev = cdev;
2247 + c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn);
2248 + if (c3cn->tid < 0)
2252 + c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev);
2256 + skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL);
2260 + skb->sk = (struct sock *)c3cn;
2261 + set_arp_failure_handler(skb, act_open_req_arp_failure);
2265 + init_offload_conn(c3cn, cdev, dst);
2267 + c3cn_reset_flag(c3cn, C3CN_DONE);
2269 + mk_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
2270 + l2t_send(cdev, skb, c3cn->l2t);
2274 + l2t_release(L2DATA(cdev), c3cn->l2t);
2276 + free_atid(cdev, c3cn->tid);
2283 + * Close a connection by sending a CPL_CLOSE_CON_REQ message. Cannot fail
2284 + * under any circumstances. We take the easy way out and always queue the
2285 + * message to the write_queue. We can optimize the case where the queue is
2286 + * already empty though the optimization is probably not worth it.
2288 +static void mk_close_req(struct s3_conn *c3cn)
2290 + struct sk_buff *skb;
2291 + struct cpl_close_con_req *req;
2292 + unsigned int tid = c3cn->tid;
2294 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2295 + c3cn, c3cn->state, c3cn->flags);
2297 + skb = alloc_skb(sizeof(struct cpl_close_con_req),
2298 + GFP_KERNEL | __GFP_NOFAIL);
2299 + req = (struct cpl_close_con_req *)__skb_put(skb, sizeof(*req));
2300 + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
2301 + req->wr.wr_lo = htonl(V_WR_TID(tid));
2302 + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
2303 + req->rsvd = htonl(c3cn->write_seq);
2305 + skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND);
2306 + if (c3cn->state != C3CN_STATE_SYN_SENT)
2307 + s3_push_frames(c3cn, 1);
2310 +static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
2313 + CXGB3_SKB_CB(skb)->seq = c3cn->write_seq;
2314 + CXGB3_SKB_CB(skb)->flags = flags;
2315 + __skb_queue_tail(&c3cn->write_queue, skb);
2319 + * Send RX credits through an RX_DATA_ACK CPL message. If nofail is 0 we are
2320 + * permitted to return without sending the message in case we cannot allocate
2321 + * an sk_buff. Returns the number of credits sent.
2323 +static u32 s3_send_rx_credits(struct s3_conn *c3cn, u32 credits, u32 dack,
2326 + struct sk_buff *skb;
2327 + struct cpl_rx_data_ack *req;
2329 + skb = (nofail ? alloc_ctrl_skb(c3cn, sizeof(*req))
2330 + : alloc_skb(sizeof(*req), GFP_ATOMIC));
2334 + req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
2335 + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
2336 + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, c3cn->tid));
2337 + req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
2338 + skb->priority = CPL_PRIORITY_ACK;
2339 + cxgb3_ofld_send(c3cn->cdev, skb);
2343 +static void mk_act_open_req(struct s3_conn *c3cn, struct sk_buff *skb,
2344 + unsigned int atid, const struct l2t_entry *e)
2346 + struct cpl_act_open_req *req;
2348 + c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn, atid);
2350 + skb->priority = CPL_PRIORITY_SETUP;
2351 + req = (struct cpl_act_open_req *)__skb_put(skb, sizeof(*req));
2352 + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
2353 + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
2354 + req->local_port = c3cn->saddr.sin_port;
2355 + req->peer_port = c3cn->daddr.sin_port;
2356 + req->local_ip = c3cn->saddr.sin_addr.s_addr;
2357 + req->peer_ip = c3cn->daddr.sin_addr.s_addr;
2358 + req->opt0h = htonl(calc_opt0h(c3cn) | V_L2T_IDX(e->idx) |
2359 + V_TX_CHANNEL(e->smt_idx));
2360 + req->opt0l = htonl(calc_opt0l(c3cn));
2365 + * Definitions and declarations for CPL handler functions.
2366 + * =======================================================
2370 + * Similar to process_cpl_msg() but takes an extra connection reference around
2371 + * the call to the handler. Should be used if the handler may drop a
2372 + * connection reference.
2374 +static inline void process_cpl_msg_ref(void (*fn) (struct s3_conn *,
2375 + struct sk_buff *),
2376 + struct s3_conn *c3cn,
2377 + struct sk_buff *skb)
2380 + process_cpl_msg(fn, c3cn, skb);
2385 + * Return whether a failed active open has allocated a TID
2387 +static inline int act_open_has_tid(int status)
2389 + return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST &&
2390 + status != CPL_ERR_ARP_MISS;
2394 + * Returns true if a connection cannot accept new Rx data.
2396 +static inline int c3cn_no_receive(const struct s3_conn *c3cn)
2398 + return c3cn->shutdown & C3CN_RCV_SHUTDOWN;
2402 + * A helper function that aborts a connection and increments the given MIB
2403 + * counter. The supplied skb is used to generate the ABORT_REQ message if
2404 + * possible. Must be called with softirqs disabled.
2406 +static inline void abort_conn(struct s3_conn *c3cn,
2407 + struct sk_buff *skb)
2409 + struct sk_buff *abort_skb;
2411 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2412 + c3cn, c3cn->state, c3cn->flags);
2414 + abort_skb = __get_cpl_reply_skb(skb, sizeof(struct cpl_abort_req),
2417 + s3_send_reset(c3cn, CPL_ABORT_SEND_RST, abort_skb);
2421 + * Returns whether an ABORT_REQ_RSS message is a negative advice.
2423 +static inline int is_neg_adv_abort(unsigned int status)
2425 + return status == CPL_ERR_RTX_NEG_ADVICE ||
2426 + status == CPL_ERR_PERSIST_NEG_ADVICE;
2430 + * CPL handler functions.
2431 + * ======================
2435 + * Process a CPL_ACT_ESTABLISH message.
2437 +static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
2440 + struct cpl_act_establish *req = cplhdr(skb);
2441 + unsigned int tid = GET_TID(req);
2442 + unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
2443 + struct s3_conn *c3cn = ctx;
2444 + struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
2446 + c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
2447 + tid, c3cn, c3cn->state, c3cn->flags);
2449 + * It's OK if the TID is currently in use, the owning connection may
2450 + * have backlogged its last CPL message(s). Just take it away.
2453 + c3cn_insert_tid(cdata, c3cn, tid);
2454 + free_atid(cdev, atid);
2456 + c3cn->qset = G_QNUM(ntohl(skb->csum));
2458 + process_cpl_msg(c3cn_act_establish, c3cn, skb);
2463 + * Process an ACT_OPEN_RPL CPL message.
2465 +static int do_act_open_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
2467 + struct s3_conn *c3cn = ctx;
2468 + struct cpl_act_open_rpl *rpl = cplhdr(skb);
2470 + c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
2471 + rpl->status, c3cn, c3cn->state, c3cn->flags);
2473 + if (act_open_has_tid(rpl->status))
2474 + cxgb3_queue_tid_release(cdev, GET_TID(rpl));
2476 + process_cpl_msg_ref(active_open_failed, c3cn, skb);
2481 + * Handler RX_ISCSI_HDR CPL messages.
2483 +static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
2485 + struct s3_conn *c3cn = ctx;
2486 + process_cpl_msg(process_rx_iscsi_hdr, c3cn, skb);
2491 + * Handler for TX_DATA_ACK CPL messages.
2493 +static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
2495 + struct s3_conn *c3cn = ctx;
2497 + process_cpl_msg(wr_ack, c3cn, skb);
2502 + * Handler for PEER_CLOSE CPL messages.
2504 +static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
2506 + struct s3_conn *c3cn = ctx;
2508 + c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
2509 + c3cn, c3cn->state, c3cn->flags);
2510 + process_cpl_msg_ref(do_peer_fin, c3cn, skb);
2515 + * Handle an ABORT_REQ_RSS CPL message.
2517 +static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
2519 + const struct cpl_abort_req_rss *req = cplhdr(skb);
2520 + struct s3_conn *c3cn = ctx;
2522 + c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
2523 + c3cn, c3cn->state, c3cn->flags);
2525 + if (is_neg_adv_abort(req->status)) {
2530 + process_cpl_msg_ref(process_abort_req, c3cn, skb);
2535 + * Handle an ABORT_RPL_RSS CPL message.
2537 +static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
2539 + struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
2540 + struct s3_conn *c3cn = ctx;
2542 + c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, 0x%x, 0x%lx.\n",
2543 + rpl->status, c3cn, c3cn ? c3cn->state : 0,
2544 + c3cn ? c3cn->flags : 0UL);
2547 + * Ignore replies to post-close aborts indicating that the abort was
2548 + * requested too late. These connections are terminated when we get
2549 + * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
2550 + * arrives the TID is either no longer used or it has been recycled.
2552 + if (rpl->status == CPL_ERR_ABORT_FAILED)
2556 + * Sometimes we've already closed the connection, e.g., a post-close
2557 + * abort races with ABORT_REQ_RSS, the latter frees the connection
2558 + * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
2559 + * but FW turns the ABORT_REQ into a regular one and so we get
2560 + * ABORT_RPL_RSS with status 0 and no connection. Only on T3A.
2565 + process_cpl_msg_ref(process_abort_rpl, c3cn, skb);
2574 + * Handler for CLOSE_CON_RPL CPL messages.
2576 +static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
2579 + struct s3_conn *c3cn = ctx;
2581 + c3cn_conn_debug("rcv, c3cn 0x%p, 0x%x, 0x%lx.\n",
2582 + c3cn, c3cn->state, c3cn->flags);
2584 + process_cpl_msg_ref(process_close_con_rpl, c3cn, skb);
2589 + * Definitions and declarations for CPL message processing.
2590 + * ========================================================
2593 +static void make_established(struct s3_conn *, u32, unsigned int);
2594 +static void act_open_retry_timer(unsigned long);
2595 +static void mk_act_open_req(struct s3_conn *, struct sk_buff *,
2596 + unsigned int, const struct l2t_entry *);
2597 +static int act_open_rpl_status_to_errno(int);
2598 +static void handle_excess_rx(struct s3_conn *, struct sk_buff *);
2599 +static int abort_status_to_errno(struct s3_conn *, int, int *);
2600 +static void send_abort_rpl(struct sk_buff *, struct t3cdev *, int);
2601 +static struct sk_buff *get_cpl_reply_skb(struct sk_buff *, size_t, gfp_t);
2604 + * Dequeue and return the first unacknowledged's WR on a connections's pending
2607 +static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
2609 + struct sk_buff *skb = c3cn->wr_pending_head;
2611 + if (likely(skb)) {
2612 + /* Don't bother clearing the tail */
2613 + c3cn->wr_pending_head = (struct sk_buff *)skb->sp;
2620 + * Return the first pending WR without removing it from the list.
2622 +static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
2624 + return c3cn->wr_pending_head;
2627 +static inline void free_wr_skb(struct sk_buff *skb)
2632 +static void purge_wr_queue(struct s3_conn *c3cn)
2634 + struct sk_buff *skb;
2635 + while ((skb = dequeue_wr(c3cn)) != NULL)
2639 +static inline void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
2642 + struct cpl_abort_rpl *rpl = cplhdr(skb);
2644 + rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
2645 + rpl->wr.wr_lo = htonl(V_WR_TID(tid));
2646 + OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
2651 + * CPL message processing ...
2652 + * ==========================
2656 + * Updates connection state from an active establish CPL message. Runs with
2657 + * the connection lock held.
2659 +static void c3cn_act_establish(struct s3_conn *c3cn,
2660 + struct sk_buff *skb)
2662 + struct cpl_act_establish *req = cplhdr(skb);
2663 + u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
2665 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2666 + c3cn, c3cn->state, c3cn->flags);
2668 + if (unlikely(c3cn->state != C3CN_STATE_SYN_SENT))
2669 + printk(KERN_ERR "TID %u expected SYN_SENT, found %d\n",
2670 + c3cn->tid, c3cn->state);
2672 + c3cn->copied_seq = c3cn->rcv_wup = c3cn->rcv_nxt = rcv_isn;
2673 + make_established(c3cn, ntohl(req->snd_isn), ntohs(req->tcp_opt));
2675 + if (unlikely(c3cn_flag(c3cn, C3CN_CLOSE_NEEDED))) {
2676 + /* upper layer has requested closing */
2677 + abort_conn(c3cn, skb);
2682 + if (s3_push_frames(c3cn, 1))
2683 + cxgb3i_conn_tx_open(c3cn);
2687 + * Handle active open failures.
2689 +static void active_open_failed(struct s3_conn *c3cn,
2690 + struct sk_buff *skb)
2692 + struct cpl_act_open_rpl *rpl = cplhdr(skb);
2694 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2695 + c3cn, c3cn->state, c3cn->flags);
2697 + if (rpl->status == CPL_ERR_CONN_EXIST &&
2698 + c3cn->retry_timer.function != act_open_retry_timer) {
2699 + c3cn->retry_timer.function = act_open_retry_timer;
2700 + c3cn_reset_timer(c3cn, &c3cn->retry_timer,
2701 + jiffies + HZ / 2);
2703 + fail_act_open(c3cn, act_open_rpl_status_to_errno(rpl->status));
2708 + * Process received pdu for a connection.
2710 +static void process_rx_iscsi_hdr(struct s3_conn *c3cn,
2711 + struct sk_buff *skb)
2713 + struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
2714 + struct cpl_iscsi_hdr_norss data_cpl;
2715 + struct cpl_rx_data_ddp_norss ddp_cpl;
2716 + unsigned int hdr_len, data_len, status;
2720 + if (unlikely(c3cn_no_receive(c3cn))) {
2721 + handle_excess_rx(c3cn, skb);
2725 + CXGB3_SKB_CB(skb)->seq = ntohl(hdr_cpl->seq);
2726 + CXGB3_SKB_CB(skb)->flags = 0;
2728 + skb_reset_transport_header(skb);
2729 + __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
2731 + len = hdr_len = ntohs(hdr_cpl->len);
2732 + /* msg coalesce is off or not enough data received */
2733 + if (skb->len <= hdr_len) {
2734 + printk(KERN_ERR "%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
2735 + c3cn->cdev->name, c3cn->tid, skb->len, hdr_len);
2739 + err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
2744 + skb_ulp_mode(skb) = ULP2_FLAG_DATA_READY;
2745 + skb_ulp_pdulen(skb) = ntohs(ddp_cpl.len);
2746 + skb_ulp_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
2747 + status = ntohl(ddp_cpl.ddp_status);
2749 + if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
2750 + skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
2751 + if (status & (1 << RX_DDP_STATUS_DCRC_SHIFT))
2752 + skb_ulp_mode(skb) |= ULP2_FLAG_DCRC_ERROR;
2753 + if (status & (1 << RX_DDP_STATUS_PAD_SHIFT))
2754 + skb_ulp_mode(skb) |= ULP2_FLAG_PAD_ERROR;
2756 + if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
2757 + err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
2760 + data_len = ntohs(data_cpl.len);
2761 + len += sizeof(data_cpl) + data_len;
2762 + } else if (status & (1 << RX_DDP_STATUS_DDP_SHIFT))
2763 + skb_ulp_mode(skb) |= ULP2_FLAG_DATA_DDPED;
2765 + c3cn->rcv_nxt = ntohl(ddp_cpl.seq) + skb_ulp_pdulen(skb);
2766 + __pskb_trim(skb, len);
2767 + __skb_queue_tail(&c3cn->receive_queue, skb);
2768 + cxgb3i_conn_pdu_ready(c3cn);
2773 + s3_send_reset(c3cn, CPL_ABORT_SEND_RST, NULL);
2778 + * Process an acknowledgment of WR completion. Advance snd_una and send the
2779 + * next batch of work requests from the write queue.
2781 +static void wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
2783 + struct cpl_wr_ack *hdr = cplhdr(skb);
2784 + unsigned int credits = ntohs(hdr->credits);
2785 + u32 snd_una = ntohl(hdr->snd_una);
2787 + c3cn->wr_avail += credits;
2788 + if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
2789 + c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
2792 + struct sk_buff *p = peek_wr(c3cn);
2794 + if (unlikely(!p)) {
2795 + printk(KERN_ERR "%u WR_ACK credits for TID %u with "
2796 + "nothing pending, state %u\n",
2797 + credits, c3cn->tid, c3cn->state);
2800 + if (unlikely(credits < p->csum)) {
2801 + p->csum -= credits;
2805 + credits -= p->csum;
2810 + if (unlikely(before(snd_una, c3cn->snd_una)))
2813 + if (c3cn->snd_una != snd_una) {
2814 + c3cn->snd_una = snd_una;
2815 + dst_confirm(c3cn->dst_cache);
2816 + if (c3cn->snd_una == c3cn->snd_nxt)
2817 + c3cn_reset_flag(c3cn, C3CN_TX_WAIT_IDLE);
2820 + if (skb_queue_len(&c3cn->write_queue) && s3_push_frames(c3cn, 0))
2821 + cxgb3i_conn_tx_open(c3cn);
2827 + * Handle a peer FIN.
2829 +static void do_peer_fin(struct s3_conn *c3cn, struct sk_buff *skb)
2833 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2834 + c3cn, c3cn->state, c3cn->flags);
2836 + if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
2839 + c3cn->shutdown |= C3CN_RCV_SHUTDOWN;
2840 + c3cn_set_flag(c3cn, C3CN_DONE);
2842 + switch (c3cn->state) {
2843 + case C3CN_STATE_ESTABLISHED:
2845 + case C3CN_STATE_CLOSING:
2850 + "%s: TID %u received PEER_CLOSE in bad state %d\n",
2851 + c3cn->cdev->name, c3cn->tid, c3cn->state);
2854 + cxgb3i_conn_closing(c3cn);
2861 + * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
2862 + * request except that we need to reply to it.
2864 +static void process_abort_req(struct s3_conn *c3cn,
2865 + struct sk_buff *skb)
2867 + int rst_status = CPL_ABORT_NO_RST;
2868 + const struct cpl_abort_req_rss *req = cplhdr(skb);
2870 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2871 + c3cn, c3cn->state, c3cn->flags);
2873 + if (!c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD)) {
2874 + c3cn_set_flag(c3cn, C3CN_ABORT_REQ_RCVD);
2875 + c3cn_set_flag(c3cn, C3CN_ABORT_SHUTDOWN);
2879 + c3cn_reset_flag(c3cn, C3CN_ABORT_REQ_RCVD);
2882 + * Three cases to consider:
2883 + * a) We haven't sent an abort_req; close the connection.
2884 + * b) We have sent a post-close abort_req that will get to TP too late
2885 + * and will generate a CPL_ERR_ABORT_FAILED reply. The reply will
2886 + * be ignored and the connection should be closed now.
2887 + * c) We have sent a regular abort_req that will get to TP too late.
2888 + * That will generate an abort_rpl with status 0, wait for it.
2890 + send_abort_rpl(skb, c3cn->cdev, rst_status);
2892 + if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
2894 + abort_status_to_errno(c3cn, req->status, &rst_status);
2901 + * Process abort replies. We only process these messages if we anticipate
2902 + * them as the coordination between SW and HW in this area is somewhat lacking
2903 + * and sometimes we get ABORT_RPLs after we are done with the connection that
2904 + * originated the ABORT_REQ.
2906 +static void process_abort_rpl(struct s3_conn *c3cn,
2907 + struct sk_buff *skb)
2909 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2910 + c3cn, c3cn->state, c3cn->flags);
2912 + if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
2913 + if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_RCVD))
2914 + c3cn_set_flag(c3cn, C3CN_ABORT_RPL_RCVD);
2916 + c3cn_reset_flag(c3cn, C3CN_ABORT_RPL_RCVD);
2917 + c3cn_reset_flag(c3cn, C3CN_ABORT_RPL_PENDING);
2918 + BUG_ON(c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD));
2926 + * Process a peer ACK to our FIN.
2928 +static void process_close_con_rpl(struct s3_conn *c3cn,
2929 + struct sk_buff *skb)
2931 + struct cpl_close_con_rpl *rpl = cplhdr(skb);
2933 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2934 + c3cn, c3cn->state, c3cn->flags);
2936 + c3cn->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
2938 + if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
2941 + if (c3cn->state == C3CN_STATE_CLOSING) {
2945 + "%s: TID %u received CLOSE_CON_RPL in bad state %d\n",
2946 + c3cn->cdev->name, c3cn->tid, c3cn->state);
2952 + * Random utility functions for CPL message processing ...
2953 + * =======================================================
2957 + * find_best_mtu - find the entry in the MTU table closest to an MTU
2959 + * @mtu: the target MTU
2961 + * Returns the index of the value in the MTU table that is closest to but
2962 + * does not exceed the target MTU.
2964 +static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
2968 + while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
2973 +static unsigned int select_mss(struct s3_conn *c3cn, unsigned int pmtu)
2976 + struct dst_entry *dst = c3cn->dst_cache;
2977 + struct t3cdev *cdev = c3cn->cdev;
2978 + const struct t3c_data *td = T3C_DATA(cdev);
2979 + u16 advmss = dst_metric(dst, RTAX_ADVMSS);
2981 + if (advmss > pmtu - 40)
2982 + advmss = pmtu - 40;
2983 + if (advmss < td->mtus[0] - 40)
2984 + advmss = td->mtus[0] - 40;
2985 + idx = find_best_mtu(td, advmss + 40);
2989 +static void fail_act_open(struct s3_conn *c3cn, int errno)
2991 + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
2992 + c3cn, c3cn->state, c3cn->flags);
2994 + c3cn->err = errno;
2999 + * Assign offload parameters to some connection fields.
3001 +static void init_offload_conn(struct s3_conn *c3cn,
3002 + struct t3cdev *cdev,
3003 + struct dst_entry *dst)
3005 + BUG_ON(c3cn->cdev != cdev);
3006 + c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs;
3007 + c3cn->wr_unacked = 0;
3008 + c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
3010 + c3cn->ctrl_skb_cache = alloc_skb(CTRL_SKB_LEN, gfp_any());
3011 + reset_wr_list(c3cn);
3014 +static void act_open_retry_timer(unsigned long data)
3016 + struct sk_buff *skb;
3017 + struct s3_conn *c3cn = (struct s3_conn *)data;
3019 + c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
3021 + spin_lock(&c3cn->lock);
3022 + skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_ATOMIC);
3024 + fail_act_open(c3cn, ENOMEM);
3026 + skb->sk = (struct sock *)c3cn;
3027 + set_arp_failure_handler(skb, act_open_req_arp_failure);
3028 + mk_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
3029 + l2t_send(c3cn->cdev, skb, c3cn->l2t);
3031 + spin_unlock(&c3cn->lock);
3036 + * Convert an ACT_OPEN_RPL status to a Linux errno.
3038 +static int act_open_rpl_status_to_errno(int status)
3041 + case CPL_ERR_CONN_RESET:
3042 + return ECONNREFUSED;
3043 + case CPL_ERR_ARP_MISS:
3044 + return EHOSTUNREACH;
3045 + case CPL_ERR_CONN_TIMEDOUT:
3047 + case CPL_ERR_TCAM_FULL:
3049 + case CPL_ERR_CONN_EXIST:
3050 + printk(KERN_ERR "ACTIVE_OPEN_RPL: 4-tuple in use\n");
3051 + return EADDRINUSE;
3058 + * Convert the status code of an ABORT_REQ into a Linux error code. Also
3059 + * indicate whether RST should be sent in response.
3061 +static int abort_status_to_errno(struct s3_conn *c3cn,
3062 + int abort_reason, int *need_rst)
3064 + switch (abort_reason) {
3065 + case CPL_ERR_BAD_SYN: /* fall through */
3066 + case CPL_ERR_CONN_RESET:
3067 + return c3cn->state == C3CN_STATE_CLOSING ? EPIPE : ECONNRESET;
3068 + case CPL_ERR_XMIT_TIMEDOUT:
3069 + case CPL_ERR_PERSIST_TIMEDOUT:
3070 + case CPL_ERR_FINWAIT2_TIMEDOUT:
3071 + case CPL_ERR_KEEPALIVE_TIMEDOUT:
3078 +static void send_abort_rpl(struct sk_buff *skb, struct t3cdev *cdev,
3081 + struct sk_buff *reply_skb;
3082 + struct cpl_abort_req_rss *req = cplhdr(skb);
3084 + reply_skb = get_cpl_reply_skb(skb, sizeof(struct cpl_abort_rpl),
3087 + reply_skb->priority = CPL_PRIORITY_DATA;
3088 + set_abort_rpl_wr(reply_skb, GET_TID(req), rst_status);
3090 + cxgb3_ofld_send(cdev, reply_skb);
3094 + * Returns an sk_buff for a reply CPL message of size len. If the input
3095 + * sk_buff has no other users it is trimmed and reused, otherwise a new buffer
3096 + * is allocated. The input skb must be of size at least len. Note that this
3097 + * operation does not destroy the original skb data even if it decides to reuse
3100 +static struct sk_buff *get_cpl_reply_skb(struct sk_buff *skb, size_t len,
3103 + if (likely(!skb_cloned(skb))) {
3104 + BUG_ON(skb->len < len);
3105 + __skb_trim(skb, len);
3108 + skb = alloc_skb(len, gfp);
3110 + __skb_put(skb, len);
3116 + * Release resources held by an offload connection (TID, L2T entry, etc.)
3118 +static void t3_release_offload_resources(struct s3_conn *c3cn)
3120 + struct t3cdev *cdev = c3cn->cdev;
3121 + unsigned int tid = c3cn->tid;
3128 + kfree_skb(c3cn->ctrl_skb_cache);
3129 + c3cn->ctrl_skb_cache = NULL;
3131 + if (c3cn->wr_avail != c3cn->wr_max) {
3132 + purge_wr_queue(c3cn);
3133 + reset_wr_list(c3cn);
3137 + l2t_release(L2DATA(cdev), c3cn->l2t);
3141 + if (c3cn->state == C3CN_STATE_SYN_SENT) /* we have ATID */
3142 + free_atid(cdev, tid);
3143 + else { /* we have TID */
3144 + cxgb3_remove_tid(cdev, (void *)c3cn, tid);
3148 + c3cn->cdev = NULL;
3152 + * Handles Rx data that arrives in a state where the connection isn't
3153 + * accepting new data.
3155 +static void handle_excess_rx(struct s3_conn *c3cn, struct sk_buff *skb)
3157 + if (!c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN))
3158 + abort_conn(c3cn, skb);
3164 + * Like get_cpl_reply_skb() but the returned buffer starts out empty.
3166 +static struct sk_buff *__get_cpl_reply_skb(struct sk_buff *skb, size_t len,
3169 + if (likely(!skb_cloned(skb) && !skb->data_len)) {
3170 + __skb_trim(skb, 0);
3173 + skb = alloc_skb(len, gfp);
3178 + * Completes some final bits of initialization for just established connections
3179 + * and changes their state to C3CN_STATE_ESTABLISHED.
3181 + * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
3183 +static void make_established(struct s3_conn *c3cn, u32 snd_isn,
3186 + c3cn_conn_debug("c3cn 0x%p, state 0x%x.\n", c3cn, c3cn->state);
3188 + c3cn->write_seq = c3cn->snd_nxt = c3cn->snd_una = snd_isn;
3191 + * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
3192 + * pass through opt0.
3194 + if (cxgb3_rcv_win > (M_RCV_BUFSIZ << 10))
3195 + c3cn->rcv_wup -= cxgb3_rcv_win - (M_RCV_BUFSIZ << 10);
3197 + dst_confirm(c3cn->dst_cache);
3200 + c3cn_set_state(c3cn, C3CN_STATE_ESTABLISHED);
3203 +++ b/drivers/scsi/cxgb3i/cxgb3i_offload.h
3206 + * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
3208 + * Written by Dimitris Michailidis (dm@chelsio.com)
3210 + * This program is distributed in the hope that it will be useful, but WITHOUT
3211 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
3212 + * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
3213 + * release for licensing terms and conditions.
3216 +#ifndef _CXGB3I_OFFLOAD_H
3217 +#define _CXGB3I_OFFLOAD_H
3219 +#include <linux/skbuff.h>
3220 +#include <net/tcp.h>
3222 +#include "common.h"
3223 +#include "adapter.h"
3224 +#include "t3cdev.h"
3225 +#include "cxgb3_offload.h"
3227 +#define cxgb3i_log_error(fmt...) printk(KERN_ERR "cxgb3i: ERR! " fmt)
3228 +#define cxgb3i_log_warn(fmt...) printk(KERN_WARNING "cxgb3i: WARN! " fmt)
3229 +#define cxgb3i_log_info(fmt...) printk(KERN_INFO "cxgb3i: " fmt)
3231 +#ifdef __DEBUG_CXGB3I__
3232 +#define cxgb3i_log_debug(fmt, args...) \
3233 + printk(KERN_INFO "cxgb3i: %s - " fmt, __func__ , ## args)
3235 +#define cxgb3i_log_debug(fmt...)
3238 +#ifdef __DEBUG_C3CN_CONN__
3239 +#define c3cn_conn_debug cxgb3i_log_debug
3241 +#define c3cn_conn_debug(fmt...)
3245 + * Data structure to keep track of cxgb3 connection.
3248 + struct net_device *dev; /* net device of with connection */
3249 + struct t3cdev *cdev; /* adapter t3cdev for net device */
3250 + unsigned long flags; /* see c3cn_flags below */
3251 + int tid; /* ID of TCP Control Block */
3252 + int qset; /* queue Set used by connection */
3253 + int mss_idx; /* Maximum Segment Size table index */
3254 + struct l2t_entry *l2t; /* ARP resolution for offload packets */
3255 + int wr_max; /* maximum in-flight writes */
3256 + int wr_avail; /* number of writes available */
3257 + int wr_unacked; /* writes since last request for */
3258 + /* completion notification */
3259 + struct sk_buff *wr_pending_head;/* head of pending write queue */
3260 + struct sk_buff *wr_pending_tail;/* tail of pending write queue */
3261 + struct sk_buff *ctrl_skb_cache; /* single entry cached skb for */
3262 + /* short-term control operations */
3263 + spinlock_t lock; /* connection status lock */
3264 + atomic_t refcnt; /* reference count on connection */
3265 + volatile unsigned int state; /* connection state */
3266 + struct sockaddr_in saddr; /* source IP/port address */
3267 + struct sockaddr_in daddr; /* destination IP/port address */
3268 + struct dst_entry *dst_cache; /* reference to destination route */
3269 + unsigned char shutdown; /* shutdown status */
3270 + struct sk_buff_head receive_queue;/* received PDUs */
3271 + struct sk_buff_head write_queue;/* un-pushed pending writes */
3273 + struct timer_list retry_timer; /* retry timer for various operations */
3274 + int err; /* connection error status */
3275 + rwlock_t callback_lock; /* lock for opaque user context */
3276 + void *user_data; /* opaque user context */
3278 + u32 rcv_nxt; /* what we want to receive next */
3279 + u32 copied_seq; /* head of yet unread data */
3280 + u32 rcv_wup; /* rcv_nxt on last window update sent */
3281 + u32 snd_nxt; /* next sequence we send */
3282 + u32 snd_una; /* first byte we want an ack for */
3284 + u32 write_seq; /* tail+1 of data held in send buffer */
3287 +/* Flags in c3cn->shutdown */
3288 +#define C3CN_RCV_SHUTDOWN 0x1
3289 +#define C3CN_SEND_SHUTDOWN 0x2
3290 +#define C3CN_SHUTDOWN_MASK (C3CN_RCV_SHUTDOWN | C3CN_SEND_SHUTDOWN)
3293 + * connection state bitmap
3295 +#define C3CN_STATE_CLOSE 0x1
3296 +#define C3CN_STATE_SYN_SENT 0x2
3297 +#define C3CN_STATE_ESTABLISHED 0x4
3298 +#define C3CN_STATE_CLOSING 0x8
3299 +#define C3CN_STATE_ABORING 0x10
3301 +#define C3CN_STATE_MASK 0xFF
3303 +static inline unsigned int c3cn_in_state(const struct s3_conn *c3cn,
3304 + unsigned int states)
3306 + return states & c3cn->state;
3310 + * Connection flags -- many to track some close related events.
3313 + C3CN_ABORT_RPL_RCVD, /* received one ABORT_RPL_RSS message */
3314 + C3CN_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
3315 + C3CN_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
3316 + C3CN_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
3318 + C3CN_ABORT_RPL_PENDING, /* expecting an abort reply */
3319 + C3CN_CLOSE_CON_REQUESTED, /* we've sent a close_conn_req */
3320 + C3CN_TX_DATA_SENT, /* already sent a TX_DATA WR */
3321 + C3CN_CLOSE_NEEDED, /* need to be closed */
3326 + * Per adapter data. Linked off of each Ethernet device port on the adapter.
3327 + * Also available via the t3cdev structure since we have pointers to our port
3328 + * net_device's there ...
3330 +struct cxgb3i_sdev_data {
3331 + struct list_head list; /* links for list of all adapters */
3332 + struct t3cdev *cdev; /* adapter t3cdev */
3333 + struct cxgb3_client *client; /* CPL client pointer */
3334 + struct adap_ports *ports; /* array of adapter ports */
3335 + unsigned int rx_page_size; /* RX page size */
3336 + struct sk_buff_head deferq; /* queue for processing replies from */
3337 + /* worker thread context */
3338 + struct work_struct deferq_task; /* worker thread */
3340 +#define NDEV2CDATA(ndev) (*(struct cxgb3i_sdev_data **)&(ndev)->ec_ptr)
3341 +#define CXGB3_SDEV_DATA(cdev) NDEV2CDATA((cdev)->lldev)
3344 + * Primary API routines.
3346 +void cxgb3i_sdev_cleanup(void);
3347 +int cxgb3i_sdev_init(cxgb3_cpl_handler_func *);
3348 +void cxgb3i_sdev_add(struct t3cdev *, struct cxgb3_client *);
3349 +void cxgb3i_sdev_remove(struct t3cdev *);
3351 +struct s3_conn *cxgb3i_c3cn_create(void);
3352 +int cxgb3i_c3cn_connect(struct s3_conn *, struct sockaddr_in *);
3353 +void cxgb3i_c3cn_rx_credits(struct s3_conn *, int);
3354 +int cxgb3i_c3cn_send_pdus(struct s3_conn *, struct sk_buff *, int);
3355 +void cxgb3i_c3cn_release(struct s3_conn *);
3358 + * Definitions for sk_buff state and ULP mode management.
3361 +struct cxgb3_skb_cb {
3362 + __u8 flags; /* see C3CB_FLAG_* below */
3363 + __u8 ulp_mode; /* ULP mode/submode of sk_buff */
3364 + __u32 seq; /* sequence number */
3365 + __u32 ddigest; /* ULP rx_data_ddp selected field */
3366 + __u32 pdulen; /* ULP rx_data_ddp selected field */
3367 + __u8 ulp_data[16]; /* scratch area for ULP */
3370 +#define CXGB3_SKB_CB(skb) ((struct cxgb3_skb_cb *)&((skb)->cb[0]))
3372 +#define skb_ulp_mode(skb) (CXGB3_SKB_CB(skb)->ulp_mode)
3373 +#define skb_ulp_ddigest(skb) (CXGB3_SKB_CB(skb)->ddigest)
3374 +#define skb_ulp_pdulen(skb) (CXGB3_SKB_CB(skb)->pdulen)
3375 +#define skb_ulp_data(skb) (CXGB3_SKB_CB(skb)->ulp_data)
3378 + C3CB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
3379 + C3CB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
3380 + C3CB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */
3381 + C3CB_FLAG_COMPL = 1 << 4, /* request WR completion */
3385 + * Top-level CPL message processing used by most CPL messages that
3386 + * pertain to connections.
3388 +static inline void process_cpl_msg(void (*fn)(struct s3_conn *,
3389 + struct sk_buff *),
3390 + struct s3_conn *c3cn,
3391 + struct sk_buff *skb)
3393 + spin_lock(&c3cn->lock);
3395 + spin_unlock(&c3cn->lock);
3399 + * Opaque version of structure the SGE stores at skb->head of TX_DATA packets
3400 + * and for which we must reserve space.
3402 +struct sge_opaque_hdr {
3404 + dma_addr_t addr[MAX_SKB_FRAGS + 1];
3407 +/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
3408 +#define TX_HEADER_LEN \
3409 + (sizeof(struct tx_data_wr) + sizeof(struct sge_opaque_hdr))
3411 +void *cxgb3i_alloc_big_mem(unsigned int);
3412 +void cxgb3i_free_big_mem(void *);
3415 + * get and set private ip for iscsi traffic
3417 +#define cxgb3i_get_private_ipv4addr(ndev) \
3418 + (((struct port_info *)(netdev_priv(ndev)))->iscsi_ipv4addr)
3419 +#define cxgb3i_set_private_ipv4addr(ndev, addr) \
3420 + (((struct port_info *)(netdev_priv(ndev)))->iscsi_ipv4addr) = addr
3422 +/* max. connections per adapter */
3423 +#define CXGB3I_MAX_CONN 16384
3424 +#endif /* _CXGB3_OFFLOAD_H */
3426 +++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
3429 + * cxgb3i_ulp2.c: Chelsio S3xx iSCSI driver.
3431 + * Copyright (c) 2008 Chelsio Communications, Inc.
3433 + * This program is free software; you can redistribute it and/or modify
3434 + * it under the terms of the GNU General Public License as published by
3435 + * the Free Software Foundation.
3437 + * Written by: Karen Xie (kxie@chelsio.com)
3440 +#include <linux/skbuff.h>
3441 +#include <scsi/scsi_cmnd.h>
3442 +#include <scsi/scsi_host.h>
3443 +#include <linux/crypto.h>
3445 +#include "cxgb3i.h"
3446 +#include "cxgb3i_ulp2.h"
3448 +#ifdef __DEBUG_CXGB3I_RX__
3449 +#define cxgb3i_rx_debug cxgb3i_log_debug
3451 +#define cxgb3i_rx_debug(fmt...)
3454 +#ifdef __DEBUG_CXGB3I_TX__
3455 +#define cxgb3i_tx_debug cxgb3i_log_debug
3457 +#define cxgb3i_tx_debug(fmt...)
3460 +#ifdef __DEBUG_CXGB3I_TAG__
3461 +#define cxgb3i_tag_debug cxgb3i_log_debug
3463 +#define cxgb3i_tag_debug(fmt...)
3466 +#ifdef __DEBUG_CXGB3I_DDP__
3467 +#define cxgb3i_ddp_debug cxgb3i_log_debug
3469 +#define cxgb3i_ddp_debug(fmt...)
3472 +static struct page *pad_page;
3474 +#define ULP2_PGIDX_MAX 4
3475 +#define ULP2_4K_PAGE_SHIFT 12
3476 +#define ULP2_4K_PAGE_MASK (~((1UL << ULP2_4K_PAGE_SHIFT) - 1))
3477 +static unsigned char ddp_page_order[ULP2_PGIDX_MAX];
3478 +static unsigned long ddp_page_size[ULP2_PGIDX_MAX];
3479 +static unsigned char ddp_page_shift[ULP2_PGIDX_MAX];
3480 +static unsigned char sw_tag_idx_bits;
3481 +static unsigned char sw_tag_age_bits;
3483 +static void cxgb3i_ddp_page_init(void)
3486 + unsigned long n = PAGE_SIZE >> ULP2_4K_PAGE_SHIFT;
3488 + if (PAGE_SIZE & (~ULP2_4K_PAGE_MASK)) {
3489 + cxgb3i_log_debug("PAGE_SIZE 0x%lx is not multiple of 4K, "
3490 + "ddp disabled.\n", PAGE_SIZE);
3493 + n = __ilog2_u32(n);
3494 + for (i = 0; i < ULP2_PGIDX_MAX; i++, n++) {
3495 + ddp_page_order[i] = n;
3496 + ddp_page_shift[i] = ULP2_4K_PAGE_SHIFT + n;
3497 + ddp_page_size[i] = 1 << ddp_page_shift[i];
3498 + cxgb3i_log_debug("%d, order %u, shift %u, size 0x%lx.\n", i,
3499 + ddp_page_order[i], ddp_page_shift[i],
3500 + ddp_page_size[i]);
3503 + sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
3504 + sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
3507 +static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
3509 + struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
3511 + req->wr.wr_lo = 0;
3512 + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
3513 + req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
3514 + V_ULPTX_CMD(ULP_MEM_WRITE));
3515 + req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) |
3516 + V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1));
3519 +static int set_ddp_map(struct cxgb3i_adapter *snic, struct pagepod_hdr *hdr,
3520 + unsigned int idx, unsigned int npods,
3521 + struct scatterlist *sgl, unsigned int sgcnt)
3523 + struct cxgb3i_ddp_info *ddp = &snic->ddp;
3524 + struct scatterlist *sg = sgl;
3525 + unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
3528 + for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
3529 + struct sk_buff *skb;
3530 + struct pagepod *ppod;
3533 + alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
3536 + cxgb3i_log_debug("skb OMM.\n");
3539 + skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
3541 + ulp_mem_io_set_hdr(skb, pm_addr);
3543 + (struct pagepod *)(skb->head + sizeof(struct ulp_mem_io));
3544 + memcpy(&(ppod->hdr), hdr, sizeof(struct pagepod));
3545 + for (j = 0, k = i * 4; j < 5; j++, k++) {
3547 + ppod->addr[j] = cpu_to_be64(sg_dma_address(sg));
3551 + ppod->addr[j] = 0UL;
3554 + skb->priority = CPL_PRIORITY_CONTROL;
3555 + cxgb3_ofld_send(snic->tdev, skb);
3560 +static int clear_ddp_map(struct cxgb3i_adapter *snic, unsigned int idx,
3561 + unsigned int npods)
3563 + struct cxgb3i_ddp_info *ddp = &snic->ddp;
3564 + unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
3567 + for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
3568 + struct sk_buff *skb;
3570 + alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
3574 + skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
3575 + memset((skb->head + sizeof(struct ulp_mem_io)), 0, PPOD_SIZE);
3576 + ulp_mem_io_set_hdr(skb, pm_addr);
3577 + skb->priority = CPL_PRIORITY_CONTROL;
3578 + cxgb3_ofld_send(snic->tdev, skb);
3583 +static int cxgb3i_ddp_sgl_check(struct scatterlist *sgl, unsigned int sgcnt)
3585 + struct scatterlist *sg;
3588 + /* make sure the sgl is fit for ddp:
3589 + * each has the same page size, and
3590 + * first & last page do not need to be used completely, and
3591 + * the rest of page must be used completely
3593 + for_each_sg(sgl, sg, sgcnt, i) {
3594 + if ((i && sg->offset) ||
3595 + ((i != sgcnt - 1) &&
3596 + (sg->length + sg->offset) != PAGE_SIZE)) {
3597 + cxgb3i_tag_debug("sg %u/%u, off %u, len %u.\n",
3598 + i, sgcnt, sg->offset, sg->length);
3606 +static inline int ddp_find_unused_entries(struct cxgb3i_ddp_info *ddp,
3607 + int start, int max, int count)
3609 + unsigned int i, j;
3611 + spin_lock(&ddp->map_lock);
3612 + for (i = start; i <= max;) {
3613 + for (j = 0; j < count; j++) {
3614 + if (ddp->map[i + j])
3618 + memset(&ddp->map[i], 1, count);
3619 + spin_unlock(&ddp->map_lock);
3624 + spin_unlock(&ddp->map_lock);
3628 +static inline void ddp_unmark_entries(struct cxgb3i_ddp_info *ddp,
3629 + int start, int count)
3631 + spin_lock(&ddp->map_lock);
3632 + memset(&ddp->map[start], 0, count);
3633 + spin_unlock(&ddp->map_lock);
3636 +static inline int sgl_map(struct cxgb3i_adapter *snic,
3637 + struct scatterlist *sgl, unsigned int sgcnt)
3639 + struct scatterlist *sg;
3642 + for_each_sg(sgl, sg, sgcnt, i) {
3643 + err = pci_map_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
3645 + cxgb3i_tag_debug("sgcnt %d/%u, pci map failed %d.\n",
3653 +static inline void sgl_unmap(struct cxgb3i_adapter *snic,
3654 + struct scatterlist *sgl, unsigned int sgcnt)
3656 + struct scatterlist *sg;
3659 + for_each_sg(sgl, sg, sgcnt, i) {
3660 + if (sg_dma_address(sg))
3661 + pci_unmap_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
3667 +u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *snic, unsigned int tid,
3668 + u32 sw_tag, unsigned int xferlen,
3669 + struct scatterlist *sgl, unsigned int sgcnt)
3671 + struct cxgb3i_ddp_info *ddp = &snic->ddp;
3672 + struct pagepod_hdr hdr;
3673 + unsigned int npods;
3674 + int idx = -1, idx_max;
3678 + if (!ddp || !sgcnt || xferlen < PAGE_SIZE) {
3679 + cxgb3i_tag_debug("sgcnt %u, xferlen %u < %lu, NO DDP.\n",
3680 + sgcnt, xferlen, PAGE_SIZE);
3681 + return RESERVED_ITT;
3684 + err = cxgb3i_ddp_sgl_check(sgl, sgcnt);
3686 + cxgb3i_tag_debug("sgcnt %u, xferlen %u, SGL check fail.\n",
3688 + return RESERVED_ITT;
3691 + npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
3692 + idx_max = ddp->nppods - npods + 1;
3694 + if (ddp->idx_last == ddp->nppods)
3695 + idx = ddp_find_unused_entries(ddp, 0, idx_max, npods);
3697 + idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1, idx_max,
3699 + if ((idx < 0) && (ddp->idx_last >= npods))
3700 + idx = ddp_find_unused_entries(ddp, 0,
3701 + ddp->idx_last - npods + 1,
3705 + cxgb3i_tag_debug("sgcnt %u, xferlen %u, npods %u NO DDP.\n",
3706 + sgcnt, xferlen, npods);
3707 + return RESERVED_ITT;
3710 + err = sgl_map(snic, sgl, sgcnt);
3714 + tag = sw_tag | (idx << snic->tag_format.rsvd_shift);
3717 + hdr.vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid));
3718 + hdr.pgsz_tag_clr = htonl(tag & snic->tag_format.rsvd_tag_mask);
3719 + hdr.maxoffset = htonl(xferlen);
3720 + hdr.pgoffset = htonl(sgl->offset);
3722 + if (set_ddp_map(snic, &hdr, idx, npods, sgl, sgcnt) < 0)
3725 + ddp->idx_last = idx;
3726 + cxgb3i_tag_debug("tid 0x%x, xfer %u, 0x%x -> ddp 0x%x (0x%x, %u).\n",
3727 + tid, xferlen, sw_tag, tag, idx, npods);
3731 + sgl_unmap(snic, sgl, sgcnt);
3732 + ddp_unmark_entries(ddp, idx, npods);
3733 + return RESERVED_ITT;
3736 +void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *snic, u32 tag,
3737 + struct scatterlist *sgl, unsigned int sgcnt)
3739 + u32 idx = (tag >> snic->tag_format.rsvd_shift) &
3740 + snic->tag_format.rsvd_mask;
3741 + unsigned int npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
3743 + if (idx < snic->tag_format.rsvd_mask) {
3744 + cxgb3i_tag_debug("ddp tag 0x%x, release idx 0x%x, npods %u.\n",
3746 + clear_ddp_map(snic, idx, npods);
3747 + ddp_unmark_entries(&snic->ddp, idx, npods);
3748 + sgl_unmap(snic, sgl, sgcnt);
3752 +int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *cconn, int hcrc, int dcrc)
3754 + struct iscsi_tcp_conn *tcp_conn = cconn->conn->dd_data;
3755 + struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
3756 + struct sk_buff *skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
3757 + GFP_KERNEL | __GFP_NOFAIL);
3758 + struct cpl_set_tcb_field *req;
3759 + u32 submode = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
3761 + /* set up ulp submode and page size */
3762 + req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
3763 + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3764 + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, c3cn->tid));
3765 + req->reply = V_NO_REPLY(1);
3767 + req->word = htons(31);
3768 + req->mask = cpu_to_be64(0xFF000000);
3769 + /* the connection page size is always the same as ddp-pgsz0 */
3770 + req->val = cpu_to_be64(submode << 24);
3771 + skb->priority = CPL_PRIORITY_CONTROL;
3773 + cxgb3_ofld_send(c3cn->cdev, skb);
3777 +static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn,
3778 + struct sk_buff *skb)
3780 + struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
3781 + struct iscsi_segment *segment = &tcp_conn->in.segment;
3782 + struct iscsi_hdr *hdr = (struct iscsi_hdr *)tcp_conn->in.hdr_buf;
3783 + unsigned char *buf = (unsigned char *)hdr;
3784 + unsigned int offset = sizeof(struct iscsi_hdr);
3787 + cxgb3i_rx_debug("conn 0x%p, skb 0x%p, len %u, flag 0x%x.\n",
3788 + conn, skb, skb->len, skb_ulp_mode(skb));
3791 + err = skb_copy_bits(skb, 0, buf, sizeof(struct iscsi_hdr));
3794 + segment->copied = sizeof(struct iscsi_hdr);
3796 + if (hdr->hlength) {
3797 + unsigned int ahslen = hdr->hlength << 2;
3798 + /* Make sure we don't overflow */
3799 + if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf))
3800 + return -ISCSI_ERR_AHSLEN;
3801 + err = skb_copy_bits(skb, offset, buf + offset, ahslen);
3806 + /* header digest */
3807 + if (conn->hdrdgst_en)
3808 + offset += ISCSI_DIGEST_SIZE;
3810 + /* check header digest */
3811 + segment->status = (conn->hdrdgst_en &&
3812 + (skb_ulp_mode(skb) & ULP2_FLAG_HCRC_ERROR)) ?
3813 + ISCSI_SEGMENT_DGST_ERR : 0;
3815 + hdr->itt = ntohl(hdr->itt);
3816 + segment->total_copied = segment->total_size;
3817 + tcp_conn->in.hdr = hdr;
3818 + err = iscsi_tcp_hdr_dissect(conn, hdr);
3822 + if (tcp_conn->in.datalen) {
3823 + segment = &tcp_conn->in.segment;
3824 + segment->status = (conn->datadgst_en &&
3825 + (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) ?
3826 + ISCSI_SEGMENT_DGST_ERR : 0;
3827 + if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
3828 + cxgb3i_ddp_debug("opcode 0x%x, data %u, ddp'ed.\n",
3829 + hdr->opcode & ISCSI_OPCODE_MASK,
3830 + tcp_conn->in.datalen);
3831 + segment->total_copied = segment->total_size;
3833 + offset += sizeof(struct cpl_iscsi_hdr_norss);
3835 + while (segment->total_copied < segment->total_size) {
3836 + iscsi_tcp_segment_map(segment, 1);
3837 + err = skb_copy_bits(skb, offset, segment->data,
3839 + iscsi_tcp_segment_unmap(segment);
3842 + segment->total_copied += segment->size;
3843 + offset += segment->size;
3845 + if (segment->total_copied < segment->total_size)
3846 + iscsi_tcp_segment_init_sg(segment,
3847 + sg_next(segment->sg),
3850 + err = segment->done(tcp_conn, segment);
3855 +static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
3863 + skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) | submode;
3866 +int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *conn)
3868 + struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
3869 + struct iscsi_segment *hdr_seg = &tcp_conn->out.segment;
3870 + struct iscsi_segment *data_seg = &tcp_conn->out.data_segment;
3871 + unsigned int hdrlen = hdr_seg->total_size;
3872 + unsigned int datalen = data_seg->total_size;
3873 + unsigned int padlen = iscsi_padding(datalen);
3874 + unsigned int copymax = SKB_MAX_HEAD(TX_HEADER_LEN);
3875 + unsigned int copylen;
3876 + struct sk_buff *skb;
3877 + unsigned char *dst;
3878 + int err = -EAGAIN;
3880 + if (data_seg->data && ((datalen + padlen) < copymax))
3881 + copylen = hdrlen + datalen + padlen;
3885 + /* supports max. 16K pdus, so one skb is enough to hold all the data */
3886 + skb = alloc_skb(TX_HEADER_LEN + copylen, GFP_ATOMIC);
3890 + skb_reserve(skb, TX_HEADER_LEN);
3891 + skb_put(skb, copylen);
3894 + tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
3896 + memcpy(dst, hdr_seg->data, hdrlen);
3902 + if (data_seg->data) {
3903 + /* data is in a linear buffer */
3904 + if (copylen > hdrlen) {
3905 + /* data fits in the skb's headroom */
3906 + memcpy(dst, data_seg->data, datalen);
3909 + memset(dst, 0, padlen);
3911 + unsigned int offset = 0;
3913 + struct page *page = alloc_page(GFP_ATOMIC);
3914 + int idx = skb_shinfo(skb)->nr_frags;
3915 + skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
3920 + frag->page = page;
3921 + frag->page_offset = 0;
3922 + if (datalen > PAGE_SIZE)
3923 + frag->size = PAGE_SIZE;
3925 + frag->size = datalen;
3926 + memcpy(page_address(page),
3927 + data_seg->data + offset, frag->size);
3929 + skb_shinfo(skb)->nr_frags++;
3930 + datalen -= frag->size;
3931 + offset += frag->size;
3935 + struct scatterlist *sg = data_seg->sg;
3936 + unsigned int offset = data_seg->sg_offset;
3938 + int idx = skb_shinfo(skb)->nr_frags;
3939 + skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
3940 + struct page *pg = sg_page(sg);
3944 + frag->page_offset = offset + sg->offset;
3945 + frag->size = min(sg->length, datalen);
3948 + skb_shinfo(skb)->nr_frags++;
3949 + datalen -= frag->size;
3954 + if (skb_shinfo(skb)->nr_frags) {
3956 + int idx = skb_shinfo(skb)->nr_frags;
3957 + skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
3958 + frag->page = pad_page;
3959 + frag->page_offset = 0;
3960 + frag->size = padlen;
3961 + skb_shinfo(skb)->nr_frags++;
3963 + datalen = data_seg->total_size + padlen;
3964 + skb->data_len += datalen;
3965 + skb->truesize += datalen;
3966 + skb->len += datalen;
3970 + err = cxgb3i_c3cn_send_pdus((struct s3_conn *)tcp_conn->sock,
3971 + skb, MSG_DONTWAIT | MSG_NOSIGNAL);
3973 + int pdulen = hdrlen + datalen + padlen;
3974 + if (conn->hdrdgst_en)
3975 + pdulen += ISCSI_DIGEST_SIZE;
3976 + if (datalen && conn->datadgst_en)
3977 + pdulen += ISCSI_DIGEST_SIZE;
3979 + hdr_seg->total_copied = hdr_seg->total_size;
3981 + data_seg->total_copied = data_seg->total_size;
3982 + conn->txdata_octets += pdulen;
3988 + if (err < 0 && err != -EAGAIN) {
3989 + cxgb3i_log_error("conn 0x%p, xmit err %d.\n", conn, err);
3990 + iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3996 +int cxgb3i_ulp2_init(void)
3998 + pad_page = alloc_page(GFP_KERNEL);
4001 + memset(page_address(pad_page), 0, PAGE_SIZE);
4002 + cxgb3i_ddp_page_init();
4006 +void cxgb3i_ulp2_cleanup(void)
4009 + __free_page(pad_page);
4014 +void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn)
4016 + struct sk_buff *skb;
4017 + unsigned int read = 0;
4018 + struct iscsi_conn *conn = c3cn->user_data;
4021 + cxgb3i_rx_debug("cn 0x%p.\n", c3cn);
4023 + read_lock(&c3cn->callback_lock);
4024 + if (unlikely(!conn || conn->suspend_rx)) {
4025 + cxgb3i_rx_debug("conn 0x%p, id %d, suspend_rx %lu!\n",
4026 + conn, conn ? conn->id : 0xFF,
4027 + conn ? conn->suspend_rx : 0xFF);
4028 + read_unlock(&c3cn->callback_lock);
4031 + skb = skb_peek(&c3cn->receive_queue);
4032 + while (!err && skb) {
4033 + __skb_unlink(skb, &c3cn->receive_queue);
4034 + read += skb_ulp_pdulen(skb);
4035 + err = cxgb3i_conn_read_pdu_skb(conn, skb);
4037 + skb = skb_peek(&c3cn->receive_queue);
4039 + read_unlock(&c3cn->callback_lock);
4041 + c3cn->copied_seq += read;
4042 + cxgb3i_c3cn_rx_credits(c3cn, read);
4044 + conn->rxdata_octets += read;
4047 + cxgb3i_log_info("conn 0x%p rx failed err %d.\n", conn, err);
4048 + iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
4052 +void cxgb3i_conn_tx_open(struct s3_conn *c3cn)
4054 + struct iscsi_conn *conn = c3cn->user_data;
4055 + struct iscsi_tcp_conn *tcp_conn;
4057 + cxgb3i_tx_debug("cn 0x%p.\n", c3cn);
4059 + cxgb3i_tx_debug("cn 0x%p, cid %d.\n", c3cn, conn->id);
4060 + tcp_conn = conn->dd_data;
4061 + scsi_queue_work(conn->session->host, &conn->xmitwork);
4065 +void cxgb3i_conn_closing(struct s3_conn *c3cn)
4067 + struct iscsi_conn *conn;
4069 + read_lock(&c3cn->callback_lock);
4070 + conn = c3cn->user_data;
4071 + if (conn && c3cn->state != C3CN_STATE_ESTABLISHED)
4072 + iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
4073 + read_unlock(&c3cn->callback_lock);
4076 +int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
4078 + struct t3cdev *tdev = snic->tdev;
4079 + struct cxgb3i_ddp_info *ddp = &snic->ddp;
4080 + struct ulp_iscsi_info uinfo;
4081 + unsigned int ppmax, bits, max_bits;
4084 + spin_lock_init(&ddp->map_lock);
4086 + err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
4088 + cxgb3i_log_error("%s, failed to get iscsi param err=%d.\n",
4093 + ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
4094 + max_bits = min(PPOD_IDX_MAX_SIZE,
4095 + (32 - sw_tag_idx_bits - sw_tag_age_bits));
4096 + bits = __ilog2_u32(ppmax) + 1;
4097 + if (bits > max_bits)
4099 + ppmax = (1 << bits) - 1;
4101 + snic->tx_max_size = min_t(unsigned int,
4102 + uinfo.max_txsz, ULP2_MAX_PKT_SIZE);
4103 + snic->rx_max_size = min_t(unsigned int,
4104 + uinfo.max_rxsz, ULP2_MAX_PKT_SIZE);
4106 + snic->tag_format.idx_bits = sw_tag_idx_bits;
4107 + snic->tag_format.age_bits = sw_tag_age_bits;
4108 + snic->tag_format.rsvd_bits = bits;
4109 + snic->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
4110 + snic->tag_format.rsvd_mask = (1 << snic->tag_format.rsvd_bits) - 1;
4111 + snic->tag_format.rsvd_tag_mask =
4112 + (1 << (snic->tag_format.rsvd_bits + PPOD_IDX_SHIFT)) - 1;
4114 + ddp->map = cxgb3i_alloc_big_mem(ppmax);
4116 + cxgb3i_log_warn("snic unable to alloc ddp ppod 0x%u, "
4117 + "ddp disabled.\n", ppmax);
4120 + ddp->llimit = uinfo.llimit;
4121 + ddp->ulimit = uinfo.ulimit;
4124 + snic->tag_format.rsvd_mask << snic->tag_format.rsvd_shift;
4125 + for (i = 0; i < ULP2_PGIDX_MAX; i++)
4126 + uinfo.pgsz_factor[i] = ddp_page_order[i];
4128 + uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
4130 + err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
4132 + cxgb3i_log_warn("snic unable to set iscsi param err=%d, "
4133 + "ddp disabled.\n", err);
4134 + goto free_ppod_map;
4137 + ddp->nppods = ppmax;
4138 + ddp->idx_last = ppmax;
4140 + tdev->ulp_iscsi = ddp;
4142 + cxgb3i_log_info("snic nppods %u (0x%x ~ 0x%x), rsvd shift %u, "
4143 + "bits %u, mask 0x%x, 0x%x, pkt %u,%u.\n",
4144 + ppmax, ddp->llimit, ddp->ulimit,
4145 + snic->tag_format.rsvd_shift,
4146 + snic->tag_format.rsvd_bits,
4147 + snic->tag_format.rsvd_mask, uinfo.tagmask,
4148 + snic->tx_max_size, snic->rx_max_size);
4153 + cxgb3i_free_big_mem(ddp->map);
4157 +void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *snic)
4159 + u8 *map = snic->ddp.map;
4162 + snic->tdev->ulp_iscsi = NULL;
4163 + spin_lock(&snic->lock);
4164 + snic->ddp.map = NULL;
4165 + spin_unlock(&snic->lock);
4166 + cxgb3i_free_big_mem(map);
4170 +++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
4173 + * cxgb3i_ulp2.h: Chelsio S3xx iSCSI driver.
4175 + * Copyright (c) 2008 Chelsio Communications, Inc.
4177 + * This program is free software; you can redistribute it and/or modify
4178 + * it under the terms of the GNU General Public License as published by
4179 + * the Free Software Foundation.
4181 + * Written by: Karen Xie (kxie@chelsio.com)
4184 +#ifndef __CXGB3I_ULP2_H__
4185 +#define __CXGB3I_ULP2_H__
4187 +#define ULP2_PDU_PAYLOAD_DFLT (16224 - ISCSI_PDU_HEADER_MAX)
4188 +#define PPOD_PAGES_MAX 4
4189 +#define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */
4191 +struct pagepod_hdr {
4200 + struct pagepod_hdr hdr;
4201 + u64 addr[PPOD_PAGES_MAX + 1];
4204 +#define PPOD_SIZE sizeof(struct pagepod) /* 64 */
4205 +#define PPOD_SIZE_SHIFT 6
4207 +#define PPOD_COLOR_SHIFT 0
4208 +#define PPOD_COLOR_SIZE 6
4209 +#define PPOD_COLOR_MASK ((1 << PPOD_COLOR_SIZE) - 1)
4211 +#define PPOD_IDX_SHIFT PPOD_COLOR_SIZE
4212 +#define PPOD_IDX_MAX_SIZE 24
4214 +#define S_PPOD_TID 0
4215 +#define M_PPOD_TID 0xFFFFFF
4216 +#define V_PPOD_TID(x) ((x) << S_PPOD_TID)
4218 +#define S_PPOD_VALID 24
4219 +#define V_PPOD_VALID(x) ((x) << S_PPOD_VALID)
4220 +#define F_PPOD_VALID V_PPOD_VALID(1U)
4222 +#define S_PPOD_COLOR 0
4223 +#define M_PPOD_COLOR 0x3F
4224 +#define V_PPOD_COLOR(x) ((x) << S_PPOD_COLOR)
4226 +#define S_PPOD_TAG 6
4227 +#define M_PPOD_TAG 0xFFFFFF
4228 +#define V_PPOD_TAG(x) ((x) << S_PPOD_TAG)
4230 +#define S_PPOD_PGSZ 30
4231 +#define M_PPOD_PGSZ 0x3
4232 +#define V_PPOD_PGSZ(x) ((x) << S_PPOD_PGSZ)
4234 +struct cpl_iscsi_hdr_norss {
4235 + union opcode_tid ot;
4244 +struct cpl_rx_data_ddp_norss {
4245 + union opcode_tid ot;
4254 +#define RX_DDP_STATUS_IPP_SHIFT 27 /* invalid pagepod */
4255 +#define RX_DDP_STATUS_TID_SHIFT 26 /* tid mismatch */
4256 +#define RX_DDP_STATUS_COLOR_SHIFT 25 /* color mismatch */
4257 +#define RX_DDP_STATUS_OFFSET_SHIFT 24 /* offset mismatch */
4258 +#define RX_DDP_STATUS_ULIMIT_SHIFT 23 /* ulimit error */
4259 +#define RX_DDP_STATUS_TAG_SHIFT 22 /* tag mismatch */
4260 +#define RX_DDP_STATUS_DCRC_SHIFT 21 /* dcrc error */
4261 +#define RX_DDP_STATUS_HCRC_SHIFT 20 /* hcrc error */
4262 +#define RX_DDP_STATUS_PAD_SHIFT 19 /* pad error */
4263 +#define RX_DDP_STATUS_PPP_SHIFT 18 /* pagepod parity error */
4264 +#define RX_DDP_STATUS_LLIMIT_SHIFT 17 /* llimit error */
4265 +#define RX_DDP_STATUS_DDP_SHIFT 16 /* ddp'able */
4266 +#define RX_DDP_STATUS_PMM_SHIFT 15 /* pagepod mismatch */
4268 +#define ULP2_FLAG_DATA_READY 0x1
4269 +#define ULP2_FLAG_DATA_DDPED 0x2
4270 +#define ULP2_FLAG_HCRC_ERROR 0x10
4271 +#define ULP2_FLAG_DCRC_ERROR 0x20
4272 +#define ULP2_FLAG_PAD_ERROR 0x40
4274 +#define ULP2_MAX_PKT_SIZE 16224
4276 +void cxgb3i_conn_closing(struct s3_conn *);
4277 +void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn);
4278 +void cxgb3i_conn_tx_open(struct s3_conn *c3cn);
4281 +++ b/drivers/scsi/cxgb3i/Kconfig
4283 +config SCSI_CXGB3_ISCSI
4284 + tristate "Chelsio S3xx iSCSI support"
4286 + select SCSI_ISCSI_ATTRS
4289 + This driver supports iSCSI offload for the Chelsio S3 series devices.
4291 +++ b/drivers/scsi/cxgb3i/Makefile
4293 +EXTRA_CFLAGS += -I$(TOPDIR)/drivers/net/cxgb3
4295 +cxgb3i-y := cxgb3i_init.o cxgb3i_iscsi.o cxgb3i_ulp2.o cxgb3i_offload.o
4297 +obj-$(CONFIG_SCSI_CXGB3_ISCSI) += cxgb3i.o
4298 --- a/drivers/scsi/Kconfig
4299 +++ b/drivers/scsi/Kconfig
4300 @@ -352,6 +352,8 @@ config ISCSI_TCP
4302 http://open-iscsi.org
4304 +source "drivers/scsi/cxgb3i/Kconfig"
4307 tristate "SGI WD93C93 SCSI Driver"
4308 depends on SGI_HAS_WD93 && SCSI
4309 --- a/drivers/scsi/Makefile
4310 +++ b/drivers/scsi/Makefile
4311 @@ -40,6 +40,7 @@ obj-$(CONFIG_LIBFC) += libfc/
4312 obj-$(CONFIG_FCOE) += fcoe/
4313 obj-$(CONFIG_ISCSI_TCP) += libiscsi.o iscsi_tcp.o
4314 obj-$(CONFIG_INFINIBAND_ISER) += libiscsi.o
4315 +obj-$(CONFIG_SCSI_CXGB3_ISCSI) += iscsi_tcp.o cxgb3i/
4316 obj-$(CONFIG_SCSI_A4000T) += 53c700.o a4000t.o
4317 obj-$(CONFIG_SCSI_ZORRO7XX) += 53c700.o zorro7xx.o
4318 obj-$(CONFIG_A3000_SCSI) += a3000.o wd33c93.o