1 From: Karen Xie <kxie@chelsio.com>
2 Subject: cxgb3i - added pdu size check and fixed ddp page setup
5 - Added documentation for cxgb3i driver, decribing how to use it to offload
7 - limit the transmit pdu size to be 4K aligned.
8 - added check for pdu size, if the negotiated pdu size if too big, print error
9 message to dmesg, and close the connection.
10 - fixed ddp page size setup when both header and data digests are off.
12 Signed-off-by: Karen Xie <kxie@chelsio.com>
13 Acked-by: John Jolly <jjolly@novell.com>
16 diff -uNr linux-2.6.27.7-4.orig/Documentation/scsi/cxgb3i.txt linux-2.6.27.7-4.new/Documentation/scsi/cxgb3i.txt
17 --- linux-2.6.27.7-4.orig/Documentation/scsi/cxgb3i.txt 1969-12-31 18:00:00.000000000 -0600
18 +++ linux-2.6.27.7-4.new/Documentation/scsi/cxgb3i.txt 2008-12-01 18:32:21.000000000 -0600
20 +Chelsio S3 iSCSI Driver for Linux
25 +The Chelsio T3 ASIC based Adapters (S310, S320, S302, S304, Mezz cards, etc.
26 +series of products) supports iSCSI acceleration and iSCSI Direct Data Placement
27 +(DDP) where the hardware handles the expensive byte touching operations, such
28 +as CRC computation and verification, and direct DMA to the final host memory
31 +The cxgb3i driver interfaces with open-iscsi initiator and provides the iSCSI
32 +acceleration through Chelsio hardware wherever applicable.
34 +Using the cxgb3i Driver
35 +=======================
37 +The following steps need to be taken to accelerates the open-iscsi initiator:
39 +1. a new transport class "cxgb3i" need to be specified. This is done by
40 + creating a new interface file in /etc/iscsi/ifaces.
42 + The content of the file should be in the following format:
43 + iface.transport_name = cxgb3i
44 + iface.net_ifacename = <ethX>
45 + iface.ipaddress = <iscsi ip address>
47 + * iface.ipaddress is optional, <iscsi ip address> can be either the same as
48 + the ethX's ip address or an address on the same subnet. Make sure the
49 + address is unique in the network.
51 +2. edit /etc/iscsi/iscsid.conf
52 + The default setting for MaxRecvDataSegmentLength (131072) is too big, search
53 + and replace all occurances of "xxx.iscsi.MaxRecvDataSegmentLength" to be a
54 + smaller value (8192 is recommended):
56 + discovery.sendtargets.iscsi.MaxRecvDataSegmentLength = 8192
57 + node.conn[0].iscsi.MaxRecvDataSegmentLength = 8192
59 +3. Load the cxgb3i driver: "modprobe cxgb3i"
61 + * in the case of recompiling the kernel, the cxgb3i selection is located at
63 + SCSI device support --->
64 + [*] SCSI low-level drivers --->
65 + <M> Chelsio S3xx iSCSI support
67 +4. To direct open-iscsi traffic to go through cxgb3i's accelerated path,
68 + "-I <iface file name>" option needs to be specified with most of the
69 + iscsiadm command. <iface file name> is the transport interface file created
71 diff -uNr linux-2.6.27.7-4.orig/drivers/scsi/cxgb3i/cxgb3i_iscsi.c linux-2.6.27.7-4.new/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
72 --- linux-2.6.27.7-4.orig/drivers/scsi/cxgb3i/cxgb3i_iscsi.c 2008-11-25 09:16:10.000000000 -0600
73 +++ linux-2.6.27.7-4.new/drivers/scsi/cxgb3i/cxgb3i_iscsi.c 2008-12-01 18:32:55.000000000 -0600
75 #include <scsi/scsi_transport_iscsi.h>
78 +#include "cxgb3i_ulp2.h"
80 +#define align_to_4k_boundary(n) \
82 + n = (n) & ~((1 << 12) - 1); \
85 static struct scsi_transport_template *cxgb3i_scsi_transport;
86 static struct scsi_host_template cxgb3i_host_template;
89 * Creates a new iSCSI connection instance for a given session
91 -static inline void cxgb3i_conn_max_xmit_dlength(struct iscsi_conn *conn)
92 +static inline int cxgb3i_conn_max_xmit_dlength(struct iscsi_conn *conn)
94 struct cxgb3i_conn *cconn = conn->dd_data;
95 + unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_SIZE,
96 + cconn->hba->snic->tx_max_size -
97 + ISCSI_PDU_HEADER_MAX);
99 + cxgb3i_log_debug("conn 0x%p, max xmit %u.\n",
100 + conn, conn->max_xmit_dlength);
102 if (conn->max_xmit_dlength)
103 conn->max_xmit_dlength = min_t(unsigned int,
104 - conn->max_xmit_dlength,
105 - cconn->hba->snic->tx_max_size -
106 - ISCSI_PDU_HEADER_MAX);
107 + conn->max_xmit_dlength, max);
109 - conn->max_xmit_dlength = cconn->hba->snic->tx_max_size -
110 - ISCSI_PDU_HEADER_MAX;
111 - cxgb3i_log_debug("conn 0x%p, max xmit %u.\n",
112 + conn->max_xmit_dlength = max;
114 + align_to_4k_boundary(conn->max_xmit_dlength);
116 + cxgb3i_log_debug("conn 0x%p, set max xmit %u.\n",
117 conn, conn->max_xmit_dlength);
122 -static inline void cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
123 +static inline int cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
125 struct cxgb3i_conn *cconn = conn->dd_data;
126 + unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_SIZE,
127 + cconn->hba->snic->rx_max_size -
128 + ISCSI_PDU_HEADER_MAX);
130 - if (conn->max_recv_dlength)
131 - conn->max_recv_dlength = min_t(unsigned int,
132 - conn->max_recv_dlength,
133 - cconn->hba->snic->rx_max_size -
134 - ISCSI_PDU_HEADER_MAX);
136 - conn->max_recv_dlength = cconn->hba->snic->rx_max_size -
137 - ISCSI_PDU_HEADER_MAX;
138 cxgb3i_log_debug("conn 0x%p, max recv %u.\n",
139 conn, conn->max_recv_dlength);
141 + align_to_4k_boundary(max);
143 + if (conn->max_recv_dlength) {
144 + if (conn->max_recv_dlength > max) {
145 + cxgb3i_log_error("MaxRecvDataSegmentLength %u, not supported."
146 + "Need to be <= %u.\n",
147 + conn->max_recv_dlength, max);
150 + conn->max_recv_dlength = min_t(unsigned int,
151 + conn->max_recv_dlength, max);
152 + align_to_4k_boundary(conn->max_recv_dlength);
154 + conn->max_recv_dlength = max;
156 + cxgb3i_log_debug("conn 0x%p, set max recv %u.\n",
157 + conn, conn->max_recv_dlength);
162 static struct iscsi_cls_conn *cxgb3i_conn_create(struct iscsi_cls_session
163 @@ -546,13 +575,13 @@
165 case ISCSI_PARAM_HDRDGST_EN:
166 err = iscsi_set_param(cls_conn, param, buf, buflen);
167 - if (!err && conn->hdrdgst_en)
169 cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en,
172 case ISCSI_PARAM_DATADGST_EN:
173 err = iscsi_set_param(cls_conn, param, buf, buflen);
174 - if (!err && conn->datadgst_en)
176 cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en,
179 @@ -568,11 +597,11 @@
181 case ISCSI_PARAM_MAX_RECV_DLENGTH:
182 err = iscsi_set_param(cls_conn, param, buf, buflen);
183 - cxgb3i_conn_max_recv_dlength(conn);
184 + err = cxgb3i_conn_max_recv_dlength(conn);
186 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
187 err = iscsi_set_param(cls_conn, param, buf, buflen);
188 - cxgb3i_conn_max_xmit_dlength(conn);
189 + err = cxgb3i_conn_max_xmit_dlength(conn);
192 return iscsi_set_param(cls_conn, param, buf, buflen);
193 diff -uNr linux-2.6.27.7-4.orig/drivers/scsi/cxgb3i/cxgb3i_offload.c linux-2.6.27.7-4.new/drivers/scsi/cxgb3i/cxgb3i_offload.c
194 --- linux-2.6.27.7-4.orig/drivers/scsi/cxgb3i/cxgb3i_offload.c 2008-11-25 09:16:10.000000000 -0600
195 +++ linux-2.6.27.7-4.new/drivers/scsi/cxgb3i/cxgb3i_offload.c 2008-12-01 18:32:55.000000000 -0600
197 #define c3cn_tx_debug(fmt...)
200 +#ifdef __DEBUG_C3CN_RX__
201 +#define c3cn_rx_debug cxgb3i_log_debug
203 +#define c3cn_rx_debug(fmt...)
206 /* connection flags */
207 static inline void c3cn_set_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
209 @@ -1577,6 +1583,9 @@
210 skb_ulp_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
211 status = ntohl(ddp_cpl.ddp_status);
213 + c3cn_rx_debug("skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
214 + skb, skb->len, skb_ulp_pdulen(skb), status);
216 if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
217 skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
218 if (status & (1 << RX_DDP_STATUS_DCRC_SHIFT))
219 diff -uNr linux-2.6.27.7-4.orig/drivers/scsi/cxgb3i/cxgb3i_ulp2.c linux-2.6.27.7-4.new/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
220 --- linux-2.6.27.7-4.orig/drivers/scsi/cxgb3i/cxgb3i_ulp2.c 2008-11-25 09:16:10.000000000 -0600
221 +++ linux-2.6.27.7-4.new/drivers/scsi/cxgb3i/cxgb3i_ulp2.c 2008-12-01 18:32:55.000000000 -0600
229 gl = kzalloc(sizeof(struct cxgb3i_gather_list) +
230 npages * (sizeof(dma_addr_t) + sizeof(struct page *)),
236 - if (j == gl->nelem)
237 + if (j == gl->nelem || sg->offset)
242 int idx = -1, idx_max;
245 - if (page_idx || !ddp || !sgcnt || xferlen < ULP2_DDP_THRESHOLD) {
246 + if (page_idx >= ULP2_PGIDX_MAX || !ddp || !sgcnt ||
247 + xferlen < ULP2_DDP_THRESHOLD) {
248 cxgb3i_tag_debug("pgidx %u, sgcnt %u, xfer %u/%u, NO ddp.\n",
249 page_idx, sgcnt, xferlen, ULP2_DDP_THRESHOLD);
256 cxgb3i_tag_debug("sgcnt %u, xferlen %u, npods %u NO DDP.\n",
257 sgcnt, xferlen, npods);
258 - return RESERVED_ITT;
262 if (ddp_alloc_gl_skb(ddp, idx, npods, gfp) < 0)
264 ddp_free_gl_skb(ddp, idx, npods);
266 ddp_unmark_entries(ddp, idx, npods);
273 struct sk_buff *skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
274 GFP_KERNEL | __GFP_NOFAIL);
275 struct cpl_set_tcb_field *req;
276 - u32 submode = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
277 - u32 pgcode = page_idx < ULP2_PGIDX_MAX ? page_idx : 0;
278 + u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
280 + if (page_idx < ULP2_PGIDX_MAX)
281 + val |= page_idx << 4;
283 /* set up ulp submode and page size */
284 req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
287 req->word = htons(31);
288 req->mask = cpu_to_be64(0xFF000000);
289 - req->val = cpu_to_be64(((pgcode << 4) | submode) << 24);
290 + req->val = cpu_to_be64(val << 24);
291 skb->priority = CPL_PRIORITY_CONTROL;
293 cxgb3_ofld_send(c3cn->cdev, skb);
294 @@ -461,12 +468,16 @@
295 segment->status = (conn->datadgst_en &&
296 (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) ?
297 ISCSI_SEGMENT_DGST_ERR : 0;
298 - if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED)
299 + if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
300 + cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, ddp'ed, "
302 + skb, hdr->opcode & ISCSI_OPCODE_MASK,
303 + tcp_conn->in.datalen, hdr->itt);
304 segment->total_copied = segment->total_size;
306 - cxgb3i_ddp_debug("opcode 0x%x, data %u, NOT ddp'ed, "
308 + cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, not ddp'ed, "
310 - hdr->opcode & ISCSI_OPCODE_MASK,
311 + skb, hdr->opcode & ISCSI_OPCODE_MASK,
312 tcp_conn->in.datalen, hdr->itt);
313 offset += sizeof(struct cpl_iscsi_hdr_norss);
315 diff -uNr linux-2.6.27.7-4.orig/drivers/scsi/cxgb3i/cxgb3i_ulp2.h linux-2.6.27.7-4.new/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
316 --- linux-2.6.27.7-4.orig/drivers/scsi/cxgb3i/cxgb3i_ulp2.h 2008-11-25 09:16:10.000000000 -0600
317 +++ linux-2.6.27.7-4.new/drivers/scsi/cxgb3i/cxgb3i_ulp2.h 2008-12-01 18:32:55.000000000 -0600
319 #define ULP2_FLAG_PAD_ERROR 0x40
321 #define ULP2_MAX_PKT_SIZE 16224
322 +#define ULP2_MAX_PDU_SIZE 8192
324 void cxgb3i_conn_closing(struct s3_conn *);
325 void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn);