1 Subject: cxgb3i - fixes over-run of skb MAX_SKB_FRAGS
2 From: Karen Xie <kxie@chelsio.com>
5 This patch fixes the over-run of skb's MAX_SKB_FRAGS between the cxgb3i and
6 cxgb3 driver on PPC64 systems.
8 Signed-off-by: Karen Xie <kxie@chelsio.com>
9 Signed-off-by: Chandra Seetharaman <sekharan@us.ibm.com>
10 Acked-by: Hannes Reinecke <hare@suse.de>
14 drivers/scsi/cxgb3i/cxgb3i.h | 148 ++++++++++++++
15 drivers/scsi/cxgb3i/cxgb3i_init.c | 4
16 drivers/scsi/cxgb3i/cxgb3i_iscsi.c | 138 ++++++-------
17 drivers/scsi/cxgb3i/cxgb3i_offload.c | 126 +++++++++---
18 drivers/scsi/cxgb3i/cxgb3i_offload.h | 6
19 drivers/scsi/cxgb3i/cxgb3i_ulp2.c | 353 ++++++++++++++++++++++++-----------
20 drivers/scsi/cxgb3i/cxgb3i_ulp2.h | 4
21 7 files changed, 553 insertions(+), 226 deletions(-)
23 --- a/drivers/scsi/cxgb3i/cxgb3i.h
24 +++ b/drivers/scsi/cxgb3i/cxgb3i.h
26 #define CXGB3I_MAX_TARGET CXGB3I_MAX_CONN
27 #define CXGB3I_MAX_LUN 512
28 #define ISCSI_PDU_HEADER_MAX (56 + 256) /* bhs + digests + ahs */
29 +#define ULP2_MAX_PKT_SIZE 16224
30 +#define ISCSI_PDU_NONPAYLOAD_MAX \
31 + (sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE + 2*ISCSI_DIGEST_SIZE)
32 +#define ULP2_MAX_PDU_PAYLOAD \
33 + (ULP2_MAX_PKT_SIZE - ISCSI_PDU_NONPAYLOAD_MAX)
36 struct cxgb3i_adapter;
38 @@ -53,12 +59,11 @@ struct cxgb3i_endpoint;
41 struct cxgb3i_tag_format {
42 - unsigned char idx_bits;
43 - unsigned char age_bits;
44 + unsigned char sw_bits;
45 unsigned char rsvd_bits;
46 unsigned char rsvd_shift;
47 + unsigned char filler[1];
53 @@ -95,11 +100,137 @@ struct cxgb3i_ddp_info {
56 unsigned int idx_last;
57 + unsigned char idx_bits;
58 + unsigned char filler[3];
62 struct cxgb3i_gather_list **gl_map;
63 struct sk_buff **gl_skb;
67 + * cxgb3i ddp tag are 32 bits, it consists of reserved bits used by h/w and
68 + * non-reserved bits that can be used by the iscsi s/w.
69 + * The reserved bits are identified by the rsvd_bits and rsvd_shift fields
70 + * in struct cxgb3i_tag_format.
72 + * The upper most reserved bit can be used to check if a tag is ddp tag or not:
73 + * if the bit is 0, the tag is a valid ddp tag
77 + * cxgb3i_is_ddp_tag - check if a given tag is a hw/ddp tag
78 + * @tformat: tag format information
79 + * @tag: tag to be checked
81 + * return true if the tag is a ddp tag, false otherwise.
83 +static inline int cxgb3i_is_ddp_tag(struct cxgb3i_tag_format *tformat, u32 tag)
85 + return !(tag & (1 << (tformat->rsvd_bits + tformat->rsvd_shift - 1)));
89 + * cxgb3i_sw_tag_usable - check if a given s/w tag has enough bits left for
90 + * the reserved/hw bits
91 + * @tformat: tag format information
92 + * @sw_tag: s/w tag to be checked
94 + * return true if the tag is a ddp tag, false otherwise.
96 +static inline int cxgb3i_sw_tag_usable(struct cxgb3i_tag_format *tformat,
99 + sw_tag >>= (32 - tformat->rsvd_bits);
104 + * cxgb3i_set_non_ddp_tag - mark a given s/w tag as an invalid ddp tag
105 + * @tformat: tag format information
106 + * @sw_tag: s/w tag to be checked
108 + * insert 1 at the upper most reserved bit to mark it as an invalid ddp tag.
110 +static inline u32 cxgb3i_set_non_ddp_tag(struct cxgb3i_tag_format *tformat,
113 + unsigned char shift = tformat->rsvd_bits + tformat->rsvd_shift - 1;
114 + u32 mask = (1 << shift) - 1;
116 + if (sw_tag && (sw_tag & ~mask)) {
117 + u32 v1 = sw_tag & ((1 << shift) - 1);
118 + u32 v2 = (sw_tag >> (shift - 1)) << shift;
120 + return v2 | v1 | 1 << shift;
122 + return sw_tag | 1 << shift;
126 + * cxgb3i_ddp_tag_base - shift the s/w tag bits so that reserved bits are not
128 + * @tformat: tag format information
129 + * @sw_tag: s/w tag to be checked
131 +static inline u32 cxgb3i_ddp_tag_base(struct cxgb3i_tag_format *tformat,
134 + u32 mask = (1 << tformat->rsvd_shift) - 1;
136 + if (sw_tag && (sw_tag & ~mask)) {
137 + u32 v1 = sw_tag & mask;
138 + u32 v2 = sw_tag >> tformat->rsvd_shift;
140 + v2 <<= tformat->rsvd_shift + tformat->rsvd_bits;
147 + * cxgb3i_tag_rsvd_bits - get the reserved bits used by the h/w
148 + * @tformat: tag format information
149 + * @tag: tag to be checked
151 + * return the reserved bits in the tag
153 +static inline u32 cxgb3i_tag_rsvd_bits(struct cxgb3i_tag_format *tformat,
156 + if (cxgb3i_is_ddp_tag(tformat, tag))
157 + return (tag >> tformat->rsvd_shift) & tformat->rsvd_mask;
162 + * cxgb3i_tag_nonrsvd_bits - get the non-reserved bits used by the s/w
163 + * @tformat: tag format information
164 + * @tag: tag to be checked
166 + * return the non-reserved bits in the tag.
168 +static inline u32 cxgb3i_tag_nonrsvd_bits(struct cxgb3i_tag_format *tformat,
171 + unsigned char shift = tformat->rsvd_bits + tformat->rsvd_shift - 1;
174 + if (cxgb3i_is_ddp_tag(tformat, tag)) {
175 + v1 = tag & ((1 << tformat->rsvd_shift) - 1);
176 + v2 = (tag >> (shift + 1)) << tformat->rsvd_shift;
178 + u32 mask = (1 << shift) - 1;
180 + tag &= ~(1 << shift);
182 + v2 = (tag >> 1) & ~mask;
189 * struct cxgb3i_hba - cxgb3i iscsi structure (per port)
191 @@ -146,16 +277,22 @@ struct cxgb3i_adapter {
192 * struct cxgb3i_conn - cxgb3i iscsi connection
194 * @tcp_conn: pointer to iscsi_tcp_conn structure
195 - * @listhead: list head to link elements
196 + * @list_head: list head to link elements
197 + * @cep: pointer to iscsi_endpoint structure
198 * @conn: pointer to iscsi_conn structure
199 * @hba: pointer to the hba this conn. is going through
200 + * @task_idx_bits: # of bits needed for session->cmds_max
201 + * @frags: temp. holding area for tx coalesced sg list pages.
203 +#define TX_PDU_PAGES_MAX (16384/512 + 1)
205 struct iscsi_tcp_conn tcp_conn;
206 struct list_head list_head;
207 struct cxgb3i_endpoint *cep;
208 struct iscsi_conn *conn;
209 struct cxgb3i_hba *hba;
210 + unsigned int task_idx_bits;
211 + skb_frag_t frags[TX_PDU_PAGES_MAX];
215 @@ -190,8 +327,7 @@ void cxgb3i_hba_host_remove(struct cxgb3
216 int cxgb3i_ulp2_init(void);
217 void cxgb3i_ulp2_cleanup(void);
218 int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *, int, int);
219 -void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32,
220 - struct scatterlist *, unsigned int);
221 +void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32);
222 u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *, unsigned int,
223 u32, unsigned int, struct scatterlist *,
225 --- a/drivers/scsi/cxgb3i/cxgb3i_init.c
226 +++ b/drivers/scsi/cxgb3i/cxgb3i_init.c
231 -#define DRV_MODULE_NAME "cxgb3i"
232 -#define DRV_MODULE_VERSION "0.1.0"
233 +#define DRV_MODULE_NAME "cxgb3i"
234 +#define DRV_MODULE_VERSION "0.9.0"
235 #define DRV_MODULE_RELDATE "Jun. 1, 2008"
237 static char version[] =
238 --- a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
239 +++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
241 #define cxgb3i_api_debug(fmt...)
244 -#define align_to_4k_boundary(n) \
246 - n = (n) & ~((1 << 12) - 1); \
249 + * align pdu size to multiple of 512 for better performance
251 +#define align_pdu_size(n) do { n = (n) & (~511); } while (0)
253 static struct scsi_transport_template *cxgb3i_scsi_transport;
254 static struct scsi_host_template cxgb3i_host_template;
255 @@ -102,7 +102,7 @@ void cxgb3i_adapter_remove(struct t3cdev
256 struct cxgb3i_adapter *snic;
258 /* remove from the list */
259 - read_lock(&cxgb3i_snic_rwlock);
260 + write_lock(&cxgb3i_snic_rwlock);
261 list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
262 if (snic->tdev == t3dev) {
263 list_del(&snic->list_head);
264 @@ -295,6 +295,8 @@ static void cxgb3i_ep_disconnect(struct
265 * stop the xmit path so the xmit_segment function is
268 + iscsi_suspend_tx(cconn->conn);
270 write_lock_bh(&cep->c3cn->callback_lock);
271 set_bit(ISCSI_SUSPEND_BIT, &cconn->conn->suspend_rx);
272 cep->c3cn->user_data = NULL;
273 @@ -391,20 +393,17 @@ static void cxgb3i_session_destroy(struc
274 static inline int cxgb3i_conn_max_xmit_dlength(struct iscsi_conn *conn)
276 struct cxgb3i_conn *cconn = conn->dd_data;
277 - unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_SIZE,
278 - cconn->hba->snic->tx_max_size -
279 - ISCSI_PDU_HEADER_MAX);
281 - cxgb3i_api_debug("conn 0x%p, max xmit %u.\n",
282 - conn, conn->max_xmit_dlength);
283 + unsigned int max = min_t(unsigned int,
284 + ULP2_MAX_PDU_PAYLOAD,
285 + cconn->hba->snic->tx_max_size -
286 + ISCSI_PDU_NONPAYLOAD_MAX);
288 if (conn->max_xmit_dlength)
289 conn->max_xmit_dlength = min_t(unsigned int,
290 - conn->max_xmit_dlength, max);
291 + conn->max_xmit_dlength, max);
293 conn->max_xmit_dlength = max;
295 - align_to_4k_boundary(conn->max_xmit_dlength);
296 + align_pdu_size(conn->max_xmit_dlength);
298 cxgb3i_api_debug("conn 0x%p, set max xmit %u.\n",
299 conn, conn->max_xmit_dlength);
300 @@ -415,14 +414,10 @@ static inline int cxgb3i_conn_max_xmit_d
301 static inline int cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
303 struct cxgb3i_conn *cconn = conn->dd_data;
304 - unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_SIZE,
305 - cconn->hba->snic->rx_max_size -
306 - ISCSI_PDU_HEADER_MAX);
308 - cxgb3i_api_debug("conn 0x%p, max recv %u.\n",
309 - conn, conn->max_recv_dlength);
311 - align_to_4k_boundary(max);
312 + unsigned int max = min_t(unsigned int,
313 + ULP2_MAX_PDU_PAYLOAD,
314 + cconn->hba->snic->tx_max_size -
315 + ISCSI_PDU_NONPAYLOAD_MAX);
317 if (conn->max_recv_dlength) {
318 if (conn->max_recv_dlength > max) {
319 @@ -433,9 +428,9 @@ static inline int cxgb3i_conn_max_recv_d
321 conn->max_recv_dlength = min_t(unsigned int,
322 conn->max_recv_dlength, max);
323 - align_to_4k_boundary(conn->max_recv_dlength);
325 conn->max_recv_dlength = max;
326 + align_pdu_size(conn->max_recv_dlength);
328 cxgb3i_api_debug("conn 0x%p, set max recv %u.\n",
329 conn, conn->max_recv_dlength);
330 @@ -516,12 +511,14 @@ static int cxgb3i_conn_bind(struct iscsi
334 + /* calculate the tag idx bits needed for this conn based on cmds_max */
335 + cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1;
337 - read_lock(&c3cn->callback_lock);
338 + write_lock(&c3cn->callback_lock);
339 /* mnc: TODO don't abuse iscsi_tcp fields */
340 tcp_conn->sock = (struct socket *)c3cn;
341 c3cn->user_data = conn;
342 - read_unlock(&c3cn->callback_lock);
343 + write_unlock(&c3cn->callback_lock);
345 cconn->hba = cep->hba;
347 @@ -609,11 +606,13 @@ static int cxgb3i_conn_set_param(struct
349 case ISCSI_PARAM_MAX_RECV_DLENGTH:
350 err = iscsi_set_param(cls_conn, param, buf, buflen);
351 - err = cxgb3i_conn_max_recv_dlength(conn);
353 + err = cxgb3i_conn_max_recv_dlength(conn);
355 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
356 err = iscsi_set_param(cls_conn, param, buf, buflen);
357 - err = cxgb3i_conn_max_xmit_dlength(conn);
359 + err = cxgb3i_conn_max_xmit_dlength(conn);
362 return iscsi_set_param(cls_conn, param, buf, buflen);
363 @@ -718,49 +717,23 @@ static void cxgb3i_conn_get_stats(struct
364 stats->custom[0].value = conn->eh_abort_cnt;
367 -static inline u32 tag_base(struct cxgb3i_tag_format *format,
368 - unsigned int idx, unsigned int age)
370 - u32 sw_bits = idx | (age << format->idx_bits);
371 - u32 tag = sw_bits >> format->rsvd_shift;
373 - tag <<= format->rsvd_bits + format->rsvd_shift;
374 - tag |= sw_bits & ((1 << format->rsvd_shift) - 1);
378 -static inline void cxgb3i_parse_tag(struct cxgb3i_tag_format *format,
379 - u32 tag, u32 *rsvd_bits, u32 *sw_bits)
382 - *rsvd_bits = (tag >> format->rsvd_shift) & format->rsvd_mask;
384 - *sw_bits = (tag >> (format->rsvd_shift + format->rsvd_bits))
385 - << format->rsvd_shift;
386 - *sw_bits |= tag & ((1 << format->rsvd_shift) - 1);
389 - cxgb3i_tag_debug("parse tag 0x%x, rsvd 0x%x, sw 0x%x.\n",
390 - tag, rsvd_bits ? *rsvd_bits : 0xFFFFFFFF,
391 - sw_bits ? *sw_bits : 0xFFFFFFFF);
395 static void cxgb3i_parse_itt(struct iscsi_conn *conn, itt_t itt,
398 struct cxgb3i_conn *cconn = conn->dd_data;
399 struct cxgb3i_adapter *snic = cconn->hba->snic;
403 - cxgb3i_parse_tag(&snic->tag_format, itt, NULL, &sw_bits);
404 + sw_bits = cxgb3i_tag_nonrsvd_bits(&snic->tag_format, tag);
406 - *idx = sw_bits & ISCSI_ITT_MASK;
407 + *idx = sw_bits & ((1 << cconn->task_idx_bits) - 1);
409 - *age = (sw_bits >> snic->tag_format.idx_bits) & ISCSI_AGE_MASK;
410 + *age = (sw_bits >> cconn->task_idx_bits) & ISCSI_AGE_MASK;
412 - cxgb3i_tag_debug("parse itt 0x%x, idx 0x%x, age 0x%x.\n",
413 - itt, idx ? *idx : 0xFFFFF, age ? *age : 0xFF);
414 + cxgb3i_tag_debug("parse tag 0x%x/0x%x, sw 0x%x, idx 0x%x, age 0x%x.\n",
415 + tag, itt, sw_bits, idx ? *idx : 0xFFFFF,
416 + age ? *age : 0xFF);
419 static int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
420 @@ -771,26 +744,40 @@ static int cxgb3i_reserve_itt(struct isc
421 struct cxgb3i_conn *cconn = conn->dd_data;
422 struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
423 struct cxgb3i_adapter *snic = cconn->hba->snic;
424 - u32 sw_tag = tag_base(&snic->tag_format, task->itt, sess->age);
425 + struct cxgb3i_tag_format *tformat = &snic->tag_format;
426 + u32 sw_tag = (sess->age << cconn->task_idx_bits) | task->itt;
427 u32 tag = RESERVED_ITT;
429 - if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE)) {
430 + if (sc && (scsi_bidi_cmnd(sc) ||
431 + sc->sc_data_direction == DMA_FROM_DEVICE) &&
432 + cxgb3i_sw_tag_usable(tformat, sw_tag)) {
433 struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
435 - cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag,
436 + tag = cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag,
438 scsi_in(sc)->table.sgl,
439 scsi_in(sc)->table.nents,
442 if (tag == RESERVED_ITT)
443 - tag = sw_tag | (snic->tag_format.rsvd_mask <<
444 - snic->tag_format.rsvd_shift);
445 + tag = cxgb3i_set_non_ddp_tag(tformat, sw_tag);
446 + /* the itt need to sent in big-endian order */
447 *hdr_itt = htonl(tag);
449 - cxgb3i_tag_debug("new tag 0x%x/0x%x (itt 0x%x, age 0x%x).\n",
450 - tag, *hdr_itt, task->itt, sess->age);
453 + if (sc->sc_data_direction == DMA_FROM_DEVICE)
454 + cxgb3i_tag_debug("read, len %u, tag 0x%x/0x%x "
455 + "(itt 0x%x, age 0x%x, sw 0x%x).\n",
456 + scsi_in(sc)->length, tag, *hdr_itt,
457 + task->itt, sess->age, sw_tag);
459 + cxgb3i_tag_debug("write, len %u, tag 0x%x/0x%x "
460 + "(itt 0x%x, age 0x%x, sw 0x%x).\n",
461 + scsi_out(sc)->length, tag, *hdr_itt,
462 + task->itt, sess->age, sw_tag);
464 + cxgb3i_tag_debug("ctrl, tag 0x%x/0x%x (itt 0x%x, age 0x%x, "
466 + tag, *hdr_itt, task->itt, sess->age, sw_tag);
470 @@ -800,14 +787,15 @@ static void cxgb3i_release_itt(struct is
471 struct iscsi_conn *conn = task->conn;
472 struct cxgb3i_conn *cconn = conn->dd_data;
473 struct cxgb3i_adapter *snic = cconn->hba->snic;
474 + struct cxgb3i_tag_format *tformat = &snic->tag_format;
475 u32 tag = ntohl(hdr_itt);
477 - cxgb3i_tag_debug("release tag 0x%x.\n", tag);
478 + cxgb3i_tag_debug("release %s tag 0x%x.\n", sc ? "scsi" : "ctrl", tag);
480 - if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE))
481 - cxgb3i_ddp_tag_release(snic, tag,
482 - scsi_in(sc)->table.sgl,
483 - scsi_in(sc)->table.nents);
484 + if (sc && (scsi_bidi_cmnd(sc) ||
485 + sc->sc_data_direction == DMA_FROM_DEVICE) &&
486 + cxgb3i_is_ddp_tag(tformat, tag))
487 + cxgb3i_ddp_tag_release(snic, tag);
491 @@ -820,7 +808,7 @@ static struct scsi_host_template cxgb3i_
492 .proc_name = "cxgb3i",
493 .queuecommand = iscsi_queuecommand,
494 .change_queue_depth = iscsi_change_queue_depth,
495 - .can_queue = 128 * (ISCSI_DEF_XMIT_CMDS_MAX - 1),
496 + .can_queue = CXGB3I_SCSI_QDEPTH_DFLT - 1,
497 .sg_tablesize = SG_ALL,
498 .max_sectors = 0xFFFF,
499 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
500 --- a/drivers/scsi/cxgb3i/cxgb3i_offload.c
501 +++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
503 #include "cxgb3i_ulp2.h"
505 #ifdef __DEBUG_C3CN_CONN__
506 -#define c3cn_conn_debug cxgb3i_log_debug
507 +#define c3cn_conn_debug cxgb3i_log_debug
509 #define c3cn_conn_debug(fmt...)
512 #ifdef __DEBUG_C3CN_TX__
513 -#define c3cn_tx_debug cxgb3i_log_debug
514 +#define c3cn_tx_debug cxgb3i_log_debug
516 #define c3cn_tx_debug(fmt...)
519 #ifdef __DEBUG_C3CN_RX__
520 -#define c3cn_rx_debug cxgb3i_log_debug
521 +#define c3cn_rx_debug cxgb3i_log_debug
523 #define c3cn_rx_debug(fmt...)
527 * module parameters releated to offloaded iscsi connection
529 -static int cxgb3_rcv_win = 256 * 1024;
530 +static int cxgb3_rcv_win = 128 * 1024;
531 module_param(cxgb3_rcv_win, int, 0644);
532 -MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
533 +MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=128KB)");
535 static int cxgb3_snd_win = 64 * 1024;
536 module_param(cxgb3_snd_win, int, 0644);
537 @@ -456,12 +456,9 @@ static unsigned int wrlen __read_mostly;
538 * The number of WRs needed for an skb depends on the number of fragments
539 * in the skb and whether it has any payload in its main body. This maps the
540 * length of the gather list represented by an skb into the # of necessary WRs.
542 - * The max. length of an skb is controlled by the max pdu size which is ~16K.
543 - * Also, assume the min. fragment length is the sector size (512), then add
544 - * extra fragment counts for iscsi bhs and payload padding.
545 + * The extra two fragments are for iscsi bhs and payload padding.
547 -#define SKB_WR_LIST_SIZE (16384/512 + 3)
548 +#define SKB_WR_LIST_SIZE (MAX_SKB_FRAGS + 2)
549 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
551 static void s3_init_wr_tab(unsigned int wr_len)
552 @@ -484,7 +481,7 @@ static void s3_init_wr_tab(unsigned int
554 static inline void reset_wr_list(struct s3_conn *c3cn)
556 - c3cn->wr_pending_head = NULL;
557 + c3cn->wr_pending_head = c3cn->wr_pending_tail = NULL;
561 @@ -495,7 +492,7 @@ static inline void reset_wr_list(struct
562 static inline void enqueue_wr(struct s3_conn *c3cn,
566 + skb_wr_next(skb) = NULL;
569 * We want to take an extra reference since both us and the driver
570 @@ -508,10 +505,22 @@ static inline void enqueue_wr(struct s3_
571 if (!c3cn->wr_pending_head)
572 c3cn->wr_pending_head = skb;
574 - c3cn->wr_pending_tail->sp = (void *)skb;
575 + skb_wr_next(c3cn->wr_pending_tail) = skb;
576 c3cn->wr_pending_tail = skb;
579 +static int count_pending_wrs(struct s3_conn *c3cn)
582 + const struct sk_buff *skb = c3cn->wr_pending_head;
586 + skb = skb_wr_next(skb);
591 static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
593 return c3cn->wr_pending_head;
594 @@ -528,8 +537,8 @@ static inline struct sk_buff *dequeue_wr
597 /* Don't bother clearing the tail */
598 - c3cn->wr_pending_head = (struct sk_buff *)skb->sp;
600 + c3cn->wr_pending_head = skb_wr_next(skb);
601 + skb_wr_next(skb) = NULL;
605 @@ -542,13 +551,15 @@ static void purge_wr_queue(struct s3_con
608 static inline void make_tx_data_wr(struct s3_conn *c3cn, struct sk_buff *skb,
610 + int len, int req_completion)
612 struct tx_data_wr *req;
614 skb_reset_transport_header(skb);
615 req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
616 - req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
618 + req->wr_hi = htonl((V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)) |
619 + (req_completion ? F_WR_COMPL : 0));
620 req->wr_lo = htonl(V_WR_TID(c3cn->tid));
621 req->sndseq = htonl(c3cn->snd_nxt);
622 /* len includes the length of any HW ULP additions */
623 @@ -556,11 +567,11 @@ static inline void make_tx_data_wr(struc
624 req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx));
625 /* V_TX_ULP_SUBMODE sets both the mode and submode */
626 req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) |
627 - V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
628 + V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
630 if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
631 - req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
632 - V_TX_CPU_IDX(c3cn->qset));
633 + req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
634 + V_TX_CPU_IDX(c3cn->qset));
635 /* Sendbuffer is in units of 32KB. */
636 req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
637 c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
638 @@ -591,7 +602,7 @@ static int c3cn_push_tx_frames(struct s3
640 if (unlikely(c3cn->state == C3CN_STATE_CONNECTING ||
641 c3cn->state == C3CN_STATE_CLOSE_WAIT_1 ||
642 - c3cn->state == C3CN_STATE_ABORTING)) {
643 + c3cn->state >= C3CN_STATE_ABORTING)) {
644 c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
647 @@ -626,19 +637,22 @@ static int c3cn_push_tx_frames(struct s3
648 c3cn->wr_unacked += wrs_needed;
649 enqueue_wr(c3cn, skb);
651 - if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
652 - len += ulp_extra_len(skb);
653 - make_tx_data_wr(c3cn, skb, len);
654 - c3cn->snd_nxt += len;
655 - if ((req_completion
656 - && c3cn->wr_unacked == wrs_needed)
657 - || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL)
658 - || c3cn->wr_unacked >= c3cn->wr_max / 2) {
659 - struct work_request_hdr *wr = cplhdr(skb);
660 + c3cn_tx_debug("c3cn 0x%p, enqueue, skb len %u/%u, frag %u, "
661 + "wr %d, left %u, unack %u.\n",
662 + c3cn, skb->len, skb->data_len, frags,
663 + wrs_needed, c3cn->wr_avail, c3cn->wr_unacked);
665 - wr->wr_hi |= htonl(F_WR_COMPL);
666 + if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
667 + if ((req_completion &&
668 + c3cn->wr_unacked == wrs_needed) ||
669 + (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL) ||
670 + c3cn->wr_unacked >= c3cn->wr_max / 2) {
671 + req_completion = 1;
672 c3cn->wr_unacked = 0;
674 + len += ulp_extra_len(skb);
675 + make_tx_data_wr(c3cn, skb, len, req_completion);
676 + c3cn->snd_nxt += len;
677 CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR;
680 @@ -1153,12 +1167,28 @@ static int do_iscsi_hdr(struct t3cdev *t
681 * Process an acknowledgment of WR completion. Advance snd_una and send the
682 * next batch of work requests from the write queue.
685 +static void check_wr_invariants(struct s3_conn *c3cn)
687 + int pending = count_pending_wrs(c3cn);
689 + if (unlikely(c3cn->wr_avail + pending != c3cn->wr_max))
690 + cxgb3i_log_error("TID %u: credit imbalance: avail %u, "
691 + "pending %u, total should be %u\n",
692 + c3cn->tid, c3cn->wr_avail, pending,
696 static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
698 struct cpl_wr_ack *hdr = cplhdr(skb);
699 unsigned int credits = ntohs(hdr->credits);
700 u32 snd_una = ntohl(hdr->snd_una);
702 + c3cn_tx_debug("%u WR credits, avail %u, unack %u, TID %u, state %u.\n",
703 + credits, c3cn->wr_avail, c3cn->wr_unacked,
704 + c3cn->tid, c3cn->state);
706 c3cn->wr_avail += credits;
707 if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
708 c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
709 @@ -1173,6 +1203,17 @@ static void process_wr_ack(struct s3_con
712 if (unlikely(credits < p->csum)) {
713 + struct tx_data_wr *w = cplhdr(p);
714 + cxgb3i_log_error("TID %u got %u WR credits need %u, "
715 + "len %u, main body %u, frags %u, "
716 + "seq # %u, ACK una %u, ACK nxt %u, "
717 + "WR_AVAIL %u, WRs pending %u\n",
718 + c3cn->tid, credits, p->csum, p->len,
719 + p->len - p->data_len,
720 + skb_shinfo(p)->nr_frags,
721 + ntohl(w->sndseq), snd_una,
722 + ntohl(hdr->snd_nxt), c3cn->wr_avail,
723 + count_pending_wrs(c3cn) - credits);
727 @@ -1182,8 +1223,14 @@ static void process_wr_ack(struct s3_con
731 - if (unlikely(before(snd_una, c3cn->snd_una)))
732 + check_wr_invariants(c3cn);
734 + if (unlikely(before(snd_una, c3cn->snd_una))) {
735 + cxgb3i_log_error("TID %u, unexpected sequence # %u in WR_ACK "
737 + c3cn->tid, snd_una, c3cn->snd_una);
741 if (c3cn->snd_una != snd_una) {
742 c3cn->snd_una = snd_una;
743 @@ -1454,11 +1501,14 @@ static void init_offload_conn(struct s3_
744 struct dst_entry *dst)
746 BUG_ON(c3cn->cdev != cdev);
747 - c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs;
748 + c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs - 1;
749 c3cn->wr_unacked = 0;
750 c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
754 + c3cn_conn_debug("c3cn 0x%p, wr max %u, avail %u.\n",
755 + c3cn, c3cn->wr_max, c3cn->wr_avail);
758 static int initiate_act_open(struct s3_conn *c3cn, struct net_device *dev)
759 @@ -1673,9 +1723,17 @@ int cxgb3i_c3cn_send_pdus(struct s3_conn
765 c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn, c3cn->err);
770 + if (c3cn->write_seq - c3cn->snd_una >= cxgb3_snd_win) {
771 + c3cn_tx_debug("c3cn 0x%p, snd %u - %u > %u.\n",
772 + c3cn, c3cn->write_seq, c3cn->snd_una,
778 --- a/drivers/scsi/cxgb3i/cxgb3i_offload.h
779 +++ b/drivers/scsi/cxgb3i/cxgb3i_offload.h
780 @@ -180,7 +180,7 @@ void cxgb3i_c3cn_release(struct s3_conn
781 * @seq: tcp sequence number
782 * @ddigest: pdu data digest
783 * @pdulen: recovered pdu length
784 - * @ulp_data: scratch area for ULP
785 + * @wr_next: scratch area for tx wr
787 struct cxgb3_skb_cb {
789 @@ -188,7 +188,7 @@ struct cxgb3_skb_cb {
794 + struct sk_buff *wr_next;
797 #define CXGB3_SKB_CB(skb) ((struct cxgb3_skb_cb *)&((skb)->cb[0]))
798 @@ -196,7 +196,7 @@ struct cxgb3_skb_cb {
799 #define skb_ulp_mode(skb) (CXGB3_SKB_CB(skb)->ulp_mode)
800 #define skb_ulp_ddigest(skb) (CXGB3_SKB_CB(skb)->ddigest)
801 #define skb_ulp_pdulen(skb) (CXGB3_SKB_CB(skb)->pdulen)
802 -#define skb_ulp_data(skb) (CXGB3_SKB_CB(skb)->ulp_data)
803 +#define skb_wr_next(skb) (CXGB3_SKB_CB(skb)->wr_next)
806 C3CB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
807 --- a/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
808 +++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
809 @@ -51,6 +51,7 @@ static unsigned char ddp_page_shift[ULP2
810 static unsigned char sw_tag_idx_bits;
811 static unsigned char sw_tag_age_bits;
812 static unsigned char page_idx = ULP2_PGIDX_MAX;
813 +static unsigned int skb_copymax = SKB_MAX_HEAD(TX_HEADER_LEN);
815 static void cxgb3i_ddp_page_init(void)
817 @@ -59,6 +60,10 @@ static void cxgb3i_ddp_page_init(void)
818 sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
819 sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
821 + cxgb3i_log_info("tag itt 0x%x, %u bits, age 0x%x, %u bits.\n",
822 + ISCSI_ITT_MASK, sw_tag_idx_bits,
823 + ISCSI_AGE_MASK, sw_tag_age_bits);
825 for (i = 0; i < ULP2_PGIDX_MAX; i++) {
826 if (PAGE_SIZE == (1UL << ddp_page_shift[i])) {
828 @@ -312,7 +317,6 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i
829 page_idx, sgcnt, xferlen, ULP2_DDP_THRESHOLD);
832 - return RESERVED_ITT;
834 gl = ddp_make_gl(xferlen, sgl, sgcnt, gfp);
836 @@ -322,9 +326,9 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i
839 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
840 - idx_max = ddp->nppods - npods + 1;
841 + idx_max = ddp->nppods - npods;
843 - if (ddp->idx_last == ddp->nppods)
844 + if (ddp->idx_last >= idx_max)
845 idx = ddp_find_unused_entries(ddp, 0, idx_max, npods, gl);
847 idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1, idx_max,
848 @@ -346,11 +350,12 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i
849 if (ddp_gl_map(snic->pdev, gl) < 0)
852 - tag = sw_tag | (idx << snic->tag_format.rsvd_shift);
853 + tag = cxgb3i_ddp_tag_base(&snic->tag_format, sw_tag);
854 + tag |= idx << PPOD_IDX_SHIFT;
857 hdr.vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid));
858 - hdr.pgsz_tag_clr = htonl(tag & snic->tag_format.rsvd_tag_mask);
859 + hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask);
860 hdr.maxoffset = htonl(xferlen);
861 hdr.pgoffset = htonl(gl->offset);
863 @@ -372,30 +377,35 @@ free_gl:
867 -void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *snic, u32 tag,
868 - struct scatterlist *sgl, unsigned int sgcnt)
869 +void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *snic, u32 tag)
871 - u32 idx = (tag >> snic->tag_format.rsvd_shift) &
872 - snic->tag_format.rsvd_mask;
873 + struct cxgb3i_ddp_info *ddp = snic->ddp;
876 - if (idx < snic->tag_format.rsvd_mask) {
877 - struct cxgb3i_ddp_info *ddp = snic->ddp;
879 + cxgb3i_log_error("release ddp tag 0x%x, ddp NULL.\n", tag);
883 + idx = (tag >> PPOD_IDX_SHIFT) & ddp->idx_mask;
884 + if (idx < ddp->nppods) {
885 struct cxgb3i_gather_list *gl = ddp->gl_map[idx];
888 if (!gl || !gl->nelem) {
889 - cxgb3i_log_warn("release tag 0x%x, idx 0x%x, no gl.\n",
891 + cxgb3i_log_error("release tag 0x%x, idx 0x%x, no gl.\n",
895 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
897 cxgb3i_tag_debug("ddp tag 0x%x, release idx 0x%x, npods %u.\n",
899 clear_ddp_map(snic, idx, npods);
900 ddp_unmark_entries(ddp, idx, npods);
901 ddp_gl_unmap(snic->pdev, gl);
904 + cxgb3i_log_error("ddp tag 0x%x, idx 0x%x > max 0x%x.\n",
905 + tag, idx, ddp->nppods);
908 int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *cconn, int hcrc, int dcrc)
909 @@ -403,12 +413,18 @@ int cxgb3i_conn_ulp_setup(struct cxgb3i_
910 struct iscsi_tcp_conn *tcp_conn = cconn->conn->dd_data;
911 struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
912 struct sk_buff *skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
913 - GFP_KERNEL | __GFP_NOFAIL);
915 struct cpl_set_tcb_field *req;
916 u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
921 if (page_idx < ULP2_PGIDX_MAX)
922 val |= page_idx << 4;
924 + cxgb3i_log_warn("TID 0x%x, host page 0x%lx default to 4K.\n",
925 + c3cn->tid, PAGE_SIZE);
927 /* set up ulp submode and page size */
928 req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
929 @@ -476,14 +492,14 @@ static int cxgb3i_conn_read_pdu_skb(stru
930 (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) ?
931 ISCSI_SEGMENT_DGST_ERR : 0;
932 if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
933 - cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, "
934 - "ddp'ed, itt 0x%x.\n",
935 - skb, hdr->opcode & ISCSI_OPCODE_MASK,
936 - tcp_conn->in.datalen, hdr->itt);
937 + cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, "
938 + "ddp'ed, itt 0x%x.\n",
939 + skb, hdr->opcode & ISCSI_OPCODE_MASK,
940 + tcp_conn->in.datalen, hdr->itt);
941 segment->total_copied = segment->total_size;
943 - cxgb3i_ddp_debug("skb 0x%p, opcode 0x%x, data %u, "
944 - "not ddp'ed, itt 0x%x.\n",
945 + cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, "
946 + "not ddp'ed, itt 0x%x.\n",
947 skb, hdr->opcode & ISCSI_OPCODE_MASK,
948 tcp_conn->in.datalen, hdr->itt);
949 offset += sizeof(struct cpl_iscsi_hdr_norss);
950 @@ -520,24 +536,141 @@ static inline void tx_skb_setmode(struct
951 skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) | submode;
954 +static int sg_page_coalesce(struct scatterlist *sg, unsigned int offset,
955 + unsigned int dlen, skb_frag_t *frags, int frag_max)
957 + unsigned int sglen = sg->length - offset;
958 + struct page *page = sg_page(sg);
959 + unsigned int datalen = dlen, copy;
967 + sglen = sg->length;
968 + page = sg_page(sg);
970 + copy = min(datalen, sglen);
971 + if (i && page == frags[i - 1].page &&
972 + offset + sg->offset ==
973 + frags[i - 1].page_offset + frags[i - 1].size) {
974 + frags[i - 1].size += copy;
976 + if (i >= frag_max) {
977 + cxgb3i_log_error("%s, too many pages > %u, "
978 + "dlen %u.\n", __func__,
983 + frags[i].page = page;
984 + frags[i].page_offset = sg->offset + offset;
985 + frags[i].size = copy;
996 +static int copy_frags_to_skb_pages(struct sk_buff *skb, skb_frag_t *frags,
997 + int frag_cnt, unsigned int datalen)
999 + struct page *page = NULL;
1000 + unsigned char *dp;
1001 + unsigned int pg_left = 0;
1002 + unsigned int copy_total = 0;
1005 + for (i = 0; i < frag_cnt; i++, frags++) {
1006 + while (frags->size) {
1007 + unsigned char *sp = page_address(frags->page);
1008 + unsigned int copy;
1011 + int cnt = skb_shinfo(skb)->nr_frags;
1013 + if (cnt >= MAX_SKB_FRAGS) {
1014 + cxgb3i_log_error("%s: pdu data %u.\n",
1015 + __func__, datalen);
1018 + page = alloc_page(GFP_ATOMIC);
1021 + dp = page_address(page);
1022 + pg_left = PAGE_SIZE;
1024 + copy = min(pg_left, datalen);
1025 + skb_fill_page_desc(skb, cnt, page, 0, copy);
1028 + skb->data_len += copy;
1029 + skb->truesize += copy;
1032 + copy = min(pg_left, frags->size);
1033 + memcpy(dp, sp + frags->page_offset, copy);
1035 + frags->size -= copy;
1036 + frags->page_offset += copy;
1039 + copy_total += copy;
1043 + return copy_total;
1046 int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *conn)
1048 - struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1049 + struct cxgb3i_conn *cconn = conn->dd_data;
1050 + struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn;
1051 struct iscsi_segment *hdr_seg = &tcp_conn->out.segment;
1052 struct iscsi_segment *data_seg = &tcp_conn->out.data_segment;
1053 unsigned int hdrlen = hdr_seg->total_size;
1054 unsigned int datalen = data_seg->total_size;
1055 unsigned int padlen = iscsi_padding(datalen);
1056 - unsigned int copymax = SKB_MAX_HEAD(TX_HEADER_LEN);
1057 - unsigned int copylen;
1058 + unsigned int copylen = hdrlen;
1059 + unsigned int copy_dlen = 0;
1060 struct sk_buff *skb;
1062 + int i, frag_cnt = 0;
1065 - if (data_seg->data && ((datalen + padlen) < copymax))
1066 - copylen = hdrlen + datalen + padlen;
1070 + * the whole pdu needs to fit into one skb, make sure we don't overrun
1071 + * the skb's frag_list. If there are more sg pages than MAX_SKB_FRAGS,
1072 + * we have to copy the data either to the head or newly allocated
1073 + * whole new page(s). This could happen if the sg contains a lot of
1074 + * fragmented data chunks (pages).
1077 + if (!data_seg->data) {
1078 + err = sg_page_coalesce(data_seg->sg,
1079 + data_seg->sg_offset,
1080 + data_seg->total_size,
1082 + TX_PDU_PAGES_MAX);
1087 + if (frag_cnt > MAX_SKB_FRAGS ||
1088 + (padlen && frag_cnt + 1 > MAX_SKB_FRAGS))
1089 + copy_dlen = datalen + padlen;
1091 + copy_dlen += datalen + padlen;
1094 + if (copylen + copy_dlen < skb_copymax)
1095 + copylen += copy_dlen;
1097 /* supports max. 16K pdus, so one skb is enough to hold all the data */
1098 skb = alloc_skb(TX_HEADER_LEN + copylen, GFP_ATOMIC);
1099 @@ -575,70 +708,84 @@ int cxgb3i_conn_ulp2_xmit(struct iscsi_c
1100 skb->data_len += datalen;
1101 skb->truesize += datalen;
1104 - struct scatterlist *sg = data_seg->sg;
1105 - unsigned int offset = data_seg->sg_offset;
1106 - struct page *page = sg_page(sg);
1107 - unsigned int sglen = sg->length - offset;
1110 - int i = skb_shinfo(skb)->nr_frags;
1111 - unsigned int copy;
1112 + } else if (copy_dlen) {
1113 + /* need to copy the page fragments */
1114 + if (copylen > hdrlen) {
1115 + skb_frag_t *frag = cconn->frags;
1119 - page = sg_page(sg);
1121 - sglen = sg->length;
1122 + /* data fits in the skb's headroom */
1123 + for (i = 0; i < frag_cnt; i++, frag++) {
1125 + page_address(frag->page) +
1126 + frag->page_offset,
1128 + dst += frag->size;
1130 - copy = min(sglen, datalen);
1132 - if (i && skb_can_coalesce(skb, i, page,
1133 - sg->offset + offset)) {
1134 - skb_shinfo(skb)->frags[i - 1].size += copy;
1137 - skb_fill_page_desc(skb, i, page,
1138 - sg->offset + offset, copy);
1140 + memset(dst, 0, padlen);
1142 + /* allocate pages to hold the data */
1143 + err = copy_frags_to_skb_pages(skb, cconn->frags,
1144 + frag_cnt, datalen);
1150 - skb->data_len += copy;
1151 - skb->truesize += copy;
1155 - } while (datalen);
1158 - if (padlen && skb_shinfo(skb)->nr_frags) {
1159 - int idx = skb_shinfo(skb)->nr_frags;
1160 - get_page(pad_page);
1161 - skb_fill_page_desc(skb, idx, pad_page, 0, padlen);
1162 - skb->data_len += padlen;
1163 - skb->truesize += padlen;
1164 - skb->len += padlen;
1165 + WARN_ON(err != datalen);
1169 + i = skb_shinfo(skb)->nr_frags;
1170 + frag = &skb_shinfo(skb)->frags[i];
1171 + dst = page_address(frag->page);
1173 + memset(dst + frag->page_offset + frag->size,
1175 + frag->size += padlen;
1179 + /* sg pages fit into frag_list */
1180 + for (i = 0; i < frag_cnt; i++)
1181 + get_page(cconn->frags[i].page);
1182 + memcpy(skb_shinfo(skb)->frags, cconn->frags,
1183 + sizeof(skb_frag_t) * frag_cnt);
1184 + skb_shinfo(skb)->nr_frags = frag_cnt;
1185 + skb->len += datalen;
1186 + skb->data_len += datalen;
1187 + skb->truesize += datalen;
1190 + i = skb_shinfo(skb)->nr_frags;
1191 + get_page(pad_page);
1192 + skb_fill_page_desc(skb, i, pad_page, 0, padlen);
1193 + skb->len += padlen;
1194 + skb->data_len += padlen;
1195 + skb->truesize += padlen;
1200 err = cxgb3i_c3cn_send_pdus((struct s3_conn *)tcp_conn->sock, skb);
1203 int pdulen = hdrlen + datalen + padlen;
1205 if (conn->hdrdgst_en)
1206 pdulen += ISCSI_DIGEST_SIZE;
1207 if (datalen && conn->datadgst_en)
1208 pdulen += ISCSI_DIGEST_SIZE;
1210 hdr_seg->total_copied = hdr_seg->total_size;
1212 - data_seg->total_copied = data_seg->total_size;
1213 + data_seg->total_copied = data_seg->total_size;
1214 conn->txdata_octets += pdulen;
1220 if (err < 0 && err != -EAGAIN) {
1221 - cxgb3i_log_error("conn 0x%p, xmit err %d.\n", conn, err);
1222 + cxgb3i_log_error("conn 0x%p, xmit err %d, skb len %u/%u.\n",
1223 + conn, err, skb->len, skb->data_len);
1224 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1227 @@ -652,6 +799,9 @@ int cxgb3i_ulp2_init(void)
1229 memset(page_address(pad_page), 0, PAGE_SIZE);
1230 cxgb3i_ddp_page_init();
1231 + cxgb3i_log_info("skb max. frag %u, head %u.\n",
1232 + (unsigned int)MAX_SKB_FRAGS,
1233 + (unsigned int)skb_copymax);
1237 @@ -720,7 +870,7 @@ void cxgb3i_conn_closing(struct s3_conn
1239 read_lock(&c3cn->callback_lock);
1240 conn = c3cn->user_data;
1241 - if (conn && c3cn->state != C3CN_STATE_ESTABLISHED)
1243 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1244 read_unlock(&c3cn->callback_lock);
1246 @@ -730,7 +880,7 @@ int cxgb3i_adapter_ulp_init(struct cxgb3
1247 struct t3cdev *tdev = snic->tdev;
1248 struct cxgb3i_ddp_info *ddp;
1249 struct ulp_iscsi_info uinfo;
1250 - unsigned int ppmax, bits, max_bits;
1251 + unsigned int ppmax, bits;
1254 err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
1255 @@ -740,26 +890,21 @@ int cxgb3i_adapter_ulp_init(struct cxgb3
1259 - ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
1260 - max_bits = min(PPOD_IDX_MAX_SIZE,
1261 - (32 - sw_tag_idx_bits - sw_tag_age_bits));
1262 - bits = __ilog2_u32(ppmax) + 1;
1263 - if (bits > max_bits)
1265 - ppmax = (1 << bits) - 1;
1267 snic->tx_max_size = min_t(unsigned int,
1268 uinfo.max_txsz, ULP2_MAX_PKT_SIZE);
1269 snic->rx_max_size = min_t(unsigned int,
1270 uinfo.max_rxsz, ULP2_MAX_PKT_SIZE);
1271 + cxgb3i_log_info("ddp max pkt size: %u/%u,%u, %u/%u,%u.\n",
1272 + snic->tx_max_size, uinfo.max_txsz, ULP2_MAX_PKT_SIZE,
1273 + snic->rx_max_size, uinfo.max_rxsz, ULP2_MAX_PKT_SIZE);
1275 - snic->tag_format.idx_bits = sw_tag_idx_bits;
1276 - snic->tag_format.age_bits = sw_tag_age_bits;
1277 - snic->tag_format.rsvd_bits = bits;
1278 - snic->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
1279 - snic->tag_format.rsvd_mask = (1 << snic->tag_format.rsvd_bits) - 1;
1280 - snic->tag_format.rsvd_tag_mask =
1281 - (1 << (snic->tag_format.rsvd_bits + PPOD_IDX_SHIFT)) - 1;
1282 + snic->tag_format.sw_bits = sw_tag_idx_bits + sw_tag_age_bits;
1284 + ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
1285 + bits = __ilog2_u32(ppmax) + 1;
1286 + if (bits > PPOD_IDX_MAX_SIZE)
1287 + bits = PPOD_IDX_MAX_SIZE;
1288 + ppmax = (1 << (bits - 1)) - 1;
1290 ddp = cxgb3i_alloc_big_mem(sizeof(struct cxgb3i_ddp_info) +
1292 @@ -779,12 +924,15 @@ int cxgb3i_adapter_ulp_init(struct cxgb3
1293 spin_lock_init(&ddp->map_lock);
1294 ddp->llimit = uinfo.llimit;
1295 ddp->ulimit = uinfo.ulimit;
1296 + ddp->nppods = ppmax;
1297 + ddp->idx_last = ppmax;
1298 + ddp->idx_bits = bits;
1299 + ddp->idx_mask = (1 << bits) - 1;
1300 + ddp->rsvd_tag_mask = (1 << (bits + PPOD_IDX_SHIFT)) - 1;
1303 - snic->tag_format.rsvd_mask << snic->tag_format.rsvd_shift;
1304 + uinfo.tagmask = ddp->idx_mask << PPOD_IDX_SHIFT;
1305 for (i = 0; i < ULP2_PGIDX_MAX; i++)
1306 uinfo.pgsz_factor[i] = ddp_page_order[i];
1308 uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
1310 err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
1311 @@ -794,19 +942,20 @@ int cxgb3i_adapter_ulp_init(struct cxgb3
1315 - ddp->nppods = ppmax;
1316 - ddp->idx_last = ppmax;
1318 tdev->ulp_iscsi = snic->ddp = ddp;
1320 - cxgb3i_log_info("snic nppods %u (0x%x ~ 0x%x), rsvd shift %u, "
1321 - "bits %u, mask 0x%x, 0x%x, pkt %u,%u.\n",
1322 - ppmax, ddp->llimit, ddp->ulimit,
1323 - snic->tag_format.rsvd_shift,
1324 - snic->tag_format.rsvd_bits,
1325 - snic->tag_format.rsvd_mask, uinfo.tagmask,
1326 - snic->tx_max_size, snic->rx_max_size);
1327 + cxgb3i_log_info("nppods %u (0x%x ~ 0x%x), bits %u, mask 0x%x,0x%x.\n",
1328 + ppmax, ddp->llimit, ddp->ulimit, ddp->idx_bits,
1329 + ddp->idx_mask, ddp->rsvd_tag_mask);
1331 + snic->tag_format.rsvd_bits = ddp->idx_bits;
1332 + snic->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
1333 + snic->tag_format.rsvd_mask = (1 << snic->tag_format.rsvd_bits) - 1;
1335 + cxgb3i_log_info("tag format: sw %u, rsvd %u,%u, mask 0x%x.\n",
1336 + snic->tag_format.sw_bits, snic->tag_format.rsvd_bits,
1337 + snic->tag_format.rsvd_shift,
1338 + snic->tag_format.rsvd_mask);
1342 --- a/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
1343 +++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.h
1345 #ifndef __CXGB3I_ULP2_H__
1346 #define __CXGB3I_ULP2_H__
1348 -#define ULP2_PDU_PAYLOAD_DFLT (16224 - ISCSI_PDU_HEADER_MAX)
1349 #define PPOD_PAGES_MAX 4
1350 #define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */
1352 @@ -100,9 +99,6 @@ struct cpl_rx_data_ddp_norss {
1353 #define ULP2_FLAG_DCRC_ERROR 0x20
1354 #define ULP2_FLAG_PAD_ERROR 0x40
1356 -#define ULP2_MAX_PKT_SIZE 16224
1357 -#define ULP2_MAX_PDU_SIZE 8192
1359 void cxgb3i_conn_closing(struct s3_conn *);
1360 void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn);
1361 void cxgb3i_conn_tx_open(struct s3_conn *c3cn);