1 Subject: cxgb3i - fixed offload wr array size
2 From: Karen Xie <kxie@chelsio.com>
5 On systems with 64K page size, the size of the wr request array used for calculating # of wrs required per pdu could be too small when scsi buffers are scattered on sector (512 bytes) boundary. This could cause the transmit to stop.
7 This patch changed the wr array size to be based on the sector size (512 bytes) instead of page size. It also added logic to coalesce scsi buffers as much as possible when constructing data pdus.
9 Signed-off-by: Karen Xie <kxie@chelsio.com>
10 Signed-off-by: Chandra Seetharaman <sekharan@us.ibm.com>
11 Signed-off-by: Hannes Reinecke <hare@suse.de>
13 diff --git a/drivers/scsi/cxgb3i/cxgb3i.h b/drivers/scsi/cxgb3i/cxgb3i.h
14 index e3f591f..48bc2ed 100644
15 --- a/drivers/scsi/cxgb3i/cxgb3i.h
16 +++ b/drivers/scsi/cxgb3i/cxgb3i.h
17 @@ -62,6 +62,25 @@ struct cxgb3i_tag_format {
21 + * struct cxgb3i_gather_list - cxgb3i direct data placement memory
24 + * @length: total data buffer length
25 + * @offset: initial offset to the 1st page
26 + * @nelem: # of pages
28 + * @phys_addr: physical address
30 +struct cxgb3i_gather_list {
32 + unsigned int length;
33 + unsigned int offset;
35 + struct page **pages;
36 + dma_addr_t phys_addr[0];
40 * struct cxgb3i_ddp_info - cxgb3i direct data placement for pdu payload
42 * @llimit: lower bound of the page pod memory
43 @@ -77,7 +96,8 @@ struct cxgb3i_ddp_info {
45 unsigned int idx_last;
48 + struct cxgb3i_gather_list **gl_map;
49 + struct sk_buff **gl_skb;
53 @@ -119,7 +139,7 @@ struct cxgb3i_adapter {
54 unsigned int rx_max_size;
56 struct cxgb3i_tag_format tag_format;
57 - struct cxgb3i_ddp_info ddp;
58 + struct cxgb3i_ddp_info *ddp;
62 @@ -174,6 +194,6 @@ void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32,
63 struct scatterlist *, unsigned int);
64 u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *, unsigned int,
65 u32, unsigned int, struct scatterlist *,
68 int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *);
70 diff --git a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
71 index 768c872..bc75dff 100644
72 --- a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
73 +++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
74 @@ -723,14 +723,16 @@ static int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
75 struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
77 cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag,
78 - scsi_out(sc)->length,
79 - scsi_out(sc)->table.sgl,
80 - scsi_out(sc)->table.nents);
81 + scsi_in(sc)->length,
82 + scsi_in(sc)->table.sgl,
83 + scsi_in(sc)->table.nents,
86 if (tag == RESERVED_ITT)
87 tag = sw_tag | (snic->tag_format.rsvd_mask <<
88 snic->tag_format.rsvd_shift);
89 *hdr_itt = htonl(tag);
94 @@ -744,8 +746,8 @@ static void cxgb3i_release_itt(struct iscsi_task *task, itt_t hdr_itt)
95 hdr_itt = ntohl(hdr_itt);
96 if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE))
97 cxgb3i_ddp_tag_release(snic, hdr_itt,
98 - scsi_out(sc)->table.sgl,
99 - scsi_out(sc)->table.nents);
100 + scsi_in(sc)->table.sgl,
101 + scsi_in(sc)->table.nents);
105 diff --git a/drivers/scsi/cxgb3i/cxgb3i_offload.c b/drivers/scsi/cxgb3i/cxgb3i_offload.c
106 index 8d0b3bf..e3f3008 100644
107 --- a/drivers/scsi/cxgb3i/cxgb3i_offload.c
108 +++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
109 @@ -211,7 +211,8 @@ static unsigned int wrlen __read_mostly;
110 * in the skb and whether it has any payload in its main body. This maps the
111 * length of the gather list represented by an skb into the # of necessary WRs.
113 -static unsigned int skb_wrs[MAX_SKB_FRAGS + 2] __read_mostly;
114 +#define SKB_WR_LIST_SIZE (16384/512 + 1)
115 +static unsigned int skb_wrs[SKB_WR_LIST_SIZE + 2] __read_mostly;
117 static void s3_init_wr_tab(unsigned int wr_len)
119 @@ -220,7 +221,7 @@ static void s3_init_wr_tab(unsigned int wr_len)
120 if (skb_wrs[1]) /* already initialized */
123 - for (i = 1; i < ARRAY_SIZE(skb_wrs); i++) {
124 + for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
125 int sgl_len = (3 * i) / 2 + (i & 1);
128 @@ -582,7 +583,6 @@ static inline void make_tx_data_wr(struct s3_conn *c3cn,
129 V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
131 if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
133 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
134 V_TX_CPU_IDX(c3cn->qset));
136 @@ -825,7 +825,7 @@ static int s3_push_frames(struct s3_conn *c3cn, int req_completion)
137 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
140 - WARN_ON(frags >= ARRAY_SIZE(skb_wrs) || wrs_needed < 1);
141 + WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
143 if (c3cn->wr_avail < wrs_needed)
145 @@ -941,12 +941,23 @@ int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb, int flags)
149 + int frags = skb_shinfo(skb)->nr_frags +
150 + (skb->len != skb->data_len);
152 if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
153 c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
158 + if (frags >= SKB_WR_LIST_SIZE) {
159 + cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
160 + c3cn, skb_shinfo(skb)->nr_frags,
161 + skb->len, skb->data_len);
168 skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
169 diff --git a/drivers/scsi/cxgb3i/cxgb3i_ulp2.c b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
170 index 7b93ba0..fc1d413 100644
171 --- a/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
172 +++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
174 static struct page *pad_page;
176 #define ULP2_PGIDX_MAX 4
177 -#define ULP2_4K_PAGE_SHIFT 12
178 -#define ULP2_4K_PAGE_MASK (~((1UL << ULP2_4K_PAGE_SHIFT) - 1))
179 -static unsigned char ddp_page_order[ULP2_PGIDX_MAX];
180 -static unsigned long ddp_page_size[ULP2_PGIDX_MAX];
181 -static unsigned char ddp_page_shift[ULP2_PGIDX_MAX];
182 +#define ULP2_DDP_THRESHOLD 2048
183 +static unsigned char ddp_page_order[ULP2_PGIDX_MAX] = {0, 1, 2, 4};
184 +static unsigned char ddp_page_shift[ULP2_PGIDX_MAX] = {12, 13, 14, 16};
185 static unsigned char sw_tag_idx_bits;
186 static unsigned char sw_tag_age_bits;
187 +static unsigned char page_idx = ULP2_PGIDX_MAX;
189 static void cxgb3i_ddp_page_init(void)
192 - unsigned long n = PAGE_SIZE >> ULP2_4K_PAGE_SHIFT;
194 - if (PAGE_SIZE & (~ULP2_4K_PAGE_MASK)) {
195 - cxgb3i_log_debug("PAGE_SIZE 0x%lx is not multiple of 4K, "
196 - "ddp disabled.\n", PAGE_SIZE);
199 - n = __ilog2_u32(n);
200 - for (i = 0; i < ULP2_PGIDX_MAX; i++, n++) {
201 - ddp_page_order[i] = n;
202 - ddp_page_shift[i] = ULP2_4K_PAGE_SHIFT + n;
203 - ddp_page_size[i] = 1 << ddp_page_shift[i];
204 - cxgb3i_log_debug("%d, order %u, shift %u, size 0x%lx.\n", i,
205 - ddp_page_order[i], ddp_page_shift[i],
209 sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
210 sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
212 + for (i = 0; i < ULP2_PGIDX_MAX; i++) {
213 + if (PAGE_SIZE == (1UL << ddp_page_shift[i])) {
215 + cxgb3i_log_info("PAGE_SIZE %lu, idx %u.\n",
216 + PAGE_SIZE, page_idx);
220 + if (page_idx == ULP2_PGIDX_MAX)
221 + cxgb3i_log_info("PAGE_SIZE %lu, no match.\n", PAGE_SIZE);
224 static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
225 @@ -91,38 +85,27 @@ static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
227 static int set_ddp_map(struct cxgb3i_adapter *snic, struct pagepod_hdr *hdr,
228 unsigned int idx, unsigned int npods,
229 - struct scatterlist *sgl, unsigned int sgcnt)
230 + struct cxgb3i_gather_list *gl)
232 - struct cxgb3i_ddp_info *ddp = &snic->ddp;
233 - struct scatterlist *sg = sgl;
234 + struct cxgb3i_ddp_info *ddp = snic->ddp;
235 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
238 - for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
239 - struct sk_buff *skb;
240 + for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
241 + struct sk_buff *skb = ddp->gl_skb[idx];
242 struct pagepod *ppod;
245 - alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
248 - cxgb3i_log_debug("skb OMM.\n");
251 - skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
254 + /* hold on to the skb until we clear the ddp mapping */
257 ulp_mem_io_set_hdr(skb, pm_addr);
259 (struct pagepod *)(skb->head + sizeof(struct ulp_mem_io));
260 memcpy(&(ppod->hdr), hdr, sizeof(struct pagepod));
261 - for (j = 0, k = i * 4; j < 5; j++, k++) {
263 - ppod->addr[j] = cpu_to_be64(sg_dma_address(sg));
267 - ppod->addr[j] = 0UL;
269 + for (pidx = 4 * i, j = 0; j < 5; ++j, ++pidx)
270 + ppod->addr[j] = pidx < gl->nelem ?
271 + cpu_to_be64(gl->phys_addr[pidx]) : 0UL;
273 skb->priority = CPL_PRIORITY_CONTROL;
274 cxgb3_ofld_send(snic->tdev, skb);
275 @@ -133,18 +116,14 @@ static int set_ddp_map(struct cxgb3i_adapter *snic, struct pagepod_hdr *hdr,
276 static int clear_ddp_map(struct cxgb3i_adapter *snic, unsigned int idx,
279 - struct cxgb3i_ddp_info *ddp = &snic->ddp;
280 + struct cxgb3i_ddp_info *ddp = snic->ddp;
281 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
284 - for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
285 - struct sk_buff *skb;
287 - alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
291 - skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
292 + for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
293 + struct sk_buff *skb = ddp->gl_skb[idx];
295 + ddp->gl_skb[idx] = NULL;
296 memset((skb->head + sizeof(struct ulp_mem_io)), 0, PPOD_SIZE);
297 ulp_mem_io_set_hdr(skb, pm_addr);
298 skb->priority = CPL_PRIORITY_CONTROL;
299 @@ -153,42 +132,80 @@ static int clear_ddp_map(struct cxgb3i_adapter *snic, unsigned int idx,
303 -static int cxgb3i_ddp_sgl_check(struct scatterlist *sgl, unsigned int sgcnt)
304 +static struct cxgb3i_gather_list *ddp_make_gl(unsigned int xferlen,
305 + struct scatterlist *sgl,
306 + unsigned int sgcnt, int gfp)
308 - struct scatterlist *sg;
311 - /* make sure the sgl is fit for ddp:
312 - * each has the same page size, and
313 - * first & last page do not need to be used completely, and
314 - * the rest of page must be used completely
316 - for_each_sg(sgl, sg, sgcnt, i) {
317 - if ((i && sg->offset) ||
318 - ((i != sgcnt - 1) &&
319 - (sg->length + sg->offset) != PAGE_SIZE)) {
320 - cxgb3i_tag_debug("sg %u/%u, off %u, len %u.\n",
321 - i, sgcnt, sg->offset, sg->length);
323 + struct cxgb3i_gather_list *gl;
324 + struct scatterlist *sg = sgl;
325 + struct page *sgpage = sg_page(sg);
326 + unsigned int sglen = sg->length;
327 + unsigned int sgoffset = sg->offset;
328 + unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >>
332 + gl = kzalloc(sizeof(struct cxgb3i_gather_list) +
333 + npages * (sizeof(dma_addr_t) + sizeof(struct page *)),
338 + gl->pages = (struct page **)&gl->phys_addr[npages];
339 + gl->length = xferlen;
340 + gl->offset = sgoffset;
341 + gl->pages[0] = sgpage;
345 + struct page *page = sg_page(sg);
347 + if (sgpage == page && sg->offset == sgoffset + sglen)
348 + sglen += sg->length;
350 + /* make sure the sgl is fit for ddp:
351 + * each has the same page size, and
352 + * all of the middle pages are used completely
354 + if ((j && sgoffset) ||
355 + ((i != sgcnt - 1) &&
356 + ((sglen + sgoffset) & ~PAGE_MASK)))
360 + if (j == gl->nelem)
362 + gl->pages[j] = page;
363 + sglen = sg->length;
364 + sgoffset = sg->offset;
379 static inline int ddp_find_unused_entries(struct cxgb3i_ddp_info *ddp,
380 - int start, int max, int count)
381 + int start, int max, int count,
382 + struct cxgb3i_gather_list *gl)
386 spin_lock(&ddp->map_lock);
387 for (i = start; i <= max;) {
388 for (j = 0; j < count; j++) {
389 - if (ddp->map[i + j])
390 + if (ddp->gl_map[i + j])
394 - memset(&ddp->map[i], 1, count);
395 + for (j = 0; j < count; j++)
396 + ddp->gl_map[i + j] = gl;
397 spin_unlock(&ddp->map_lock);
400 @@ -202,86 +219,127 @@ static inline void ddp_unmark_entries(struct cxgb3i_ddp_info *ddp,
401 int start, int count)
403 spin_lock(&ddp->map_lock);
404 - memset(&ddp->map[start], 0, count);
405 + memset(&ddp->gl_map[start], 0,
406 + count * sizeof(struct cxgb3i_gather_list *));
407 spin_unlock(&ddp->map_lock);
410 -static inline int sgl_map(struct cxgb3i_adapter *snic,
411 - struct scatterlist *sgl, unsigned int sgcnt)
412 +static inline void ddp_free_gl_skb(struct cxgb3i_ddp_info *ddp,
413 + int idx, int count)
415 - struct scatterlist *sg;
419 - for_each_sg(sgl, sg, sgcnt, i) {
420 - err = pci_map_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
422 - cxgb3i_tag_debug("sgcnt %d/%u, pci map failed %d.\n",
425 + for (i = 0; i < count; i++, idx++)
426 + if (ddp->gl_skb[idx]) {
427 + kfree_skb(ddp->gl_skb[idx]);
428 + ddp->gl_skb[idx] = NULL;
432 +static inline int ddp_alloc_gl_skb(struct cxgb3i_ddp_info *ddp,
433 + int idx, int count, int gfp)
437 + for (i = 0; i < count; i++) {
438 + struct sk_buff *skb = alloc_skb(sizeof(struct ulp_mem_io) +
441 + ddp->gl_skb[idx + i] = skb;
442 + skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
444 + ddp_free_gl_skb(ddp, idx, i);
452 -static inline void sgl_unmap(struct cxgb3i_adapter *snic,
453 - struct scatterlist *sgl, unsigned int sgcnt)
454 +static inline void ddp_gl_unmap(struct pci_dev *pdev,
455 + struct cxgb3i_gather_list *gl)
457 - struct scatterlist *sg;
460 - for_each_sg(sgl, sg, sgcnt, i) {
461 - if (sg_dma_address(sg))
462 - pci_unmap_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
465 + for (i = 0; i < gl->nelem; i++)
466 + pci_unmap_page(pdev, gl->phys_addr[i], PAGE_SIZE,
467 + PCI_DMA_FROMDEVICE);
470 +static inline int ddp_gl_map(struct pci_dev *pdev,
471 + struct cxgb3i_gather_list *gl)
475 + for (i = 0; i < gl->nelem; i++) {
476 + gl->phys_addr[i] = pci_map_page(pdev, gl->pages[i], 0,
478 + PCI_DMA_FROMDEVICE);
479 + if (unlikely(pci_dma_mapping_error(pdev, gl->phys_addr[i])))
487 + unsigned int nelem = gl->nelem;
490 + ddp_gl_unmap(pdev, gl);
496 u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *snic, unsigned int tid,
497 u32 sw_tag, unsigned int xferlen,
498 - struct scatterlist *sgl, unsigned int sgcnt)
499 + struct scatterlist *sgl, unsigned int sgcnt,
502 - struct cxgb3i_ddp_info *ddp = &snic->ddp;
503 + struct cxgb3i_ddp_info *ddp = snic->ddp;
504 + struct cxgb3i_gather_list *gl;
505 struct pagepod_hdr hdr;
507 int idx = -1, idx_max;
511 - if (!ddp || !sgcnt || xferlen < PAGE_SIZE) {
512 - cxgb3i_tag_debug("sgcnt %u, xferlen %u < %lu, NO DDP.\n",
513 - sgcnt, xferlen, PAGE_SIZE);
514 + if (page_idx || !ddp || !sgcnt || xferlen < ULP2_DDP_THRESHOLD) {
515 + cxgb3i_tag_debug("pgidx %u, sgcnt %u, xfer %u/%u, NO ddp.\n",
516 + page_idx, sgcnt, xferlen, ULP2_DDP_THRESHOLD);
520 - err = cxgb3i_ddp_sgl_check(sgl, sgcnt);
522 + gl = ddp_make_gl(xferlen, sgl, sgcnt, gfp);
524 cxgb3i_tag_debug("sgcnt %u, xferlen %u, SGL check fail.\n",
529 - npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
530 + npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
531 idx_max = ddp->nppods - npods + 1;
533 if (ddp->idx_last == ddp->nppods)
534 - idx = ddp_find_unused_entries(ddp, 0, idx_max, npods);
535 + idx = ddp_find_unused_entries(ddp, 0, idx_max, npods, gl);
537 idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1, idx_max,
540 if ((idx < 0) && (ddp->idx_last >= npods))
541 idx = ddp_find_unused_entries(ddp, 0,
542 ddp->idx_last - npods + 1,
548 cxgb3i_tag_debug("sgcnt %u, xferlen %u, npods %u NO DDP.\n",
549 sgcnt, xferlen, npods);
553 - err = sgl_map(snic, sgl, sgcnt);
555 + if (ddp_alloc_gl_skb(ddp, idx, npods, gfp) < 0)
556 + goto unmark_entries;
558 + if (ddp_gl_map(snic->pdev, gl) < 0)
561 tag = sw_tag | (idx << snic->tag_format.rsvd_shift);
562 @@ -290,9 +348,9 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *snic, unsigned int tid,
563 hdr.vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid));
564 hdr.pgsz_tag_clr = htonl(tag & snic->tag_format.rsvd_tag_mask);
565 hdr.maxoffset = htonl(xferlen);
566 - hdr.pgoffset = htonl(sgl->offset);
567 + hdr.pgoffset = htonl(gl->offset);
569 - if (set_ddp_map(snic, &hdr, idx, npods, sgl, sgcnt) < 0)
570 + if (set_ddp_map(snic, &hdr, idx, npods, gl) < 0)
574 @@ -301,7 +359,9 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *snic, unsigned int tid,
578 - sgl_unmap(snic, sgl, sgcnt);
579 + ddp_gl_unmap(snic->pdev, gl);
580 + ddp_free_gl_skb(ddp, idx, npods);
582 ddp_unmark_entries(ddp, idx, npods);
585 @@ -311,14 +371,18 @@ void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *snic, u32 tag,
587 u32 idx = (tag >> snic->tag_format.rsvd_shift) &
588 snic->tag_format.rsvd_mask;
589 - unsigned int npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
591 if (idx < snic->tag_format.rsvd_mask) {
592 + struct cxgb3i_ddp_info *ddp = snic->ddp;
593 + struct cxgb3i_gather_list *gl = ddp->gl_map[idx];
594 + unsigned int npods = (gl->nelem + PPOD_PAGES_MAX - 1) >>
597 cxgb3i_tag_debug("ddp tag 0x%x, release idx 0x%x, npods %u.\n",
599 clear_ddp_map(snic, idx, npods);
600 - ddp_unmark_entries(&snic->ddp, idx, npods);
601 - sgl_unmap(snic, sgl, sgcnt);
602 + ddp_unmark_entries(ddp, idx, npods);
603 + ddp_gl_unmap(snic->pdev, gl);
607 @@ -330,6 +394,7 @@ int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *cconn, int hcrc, int dcrc)
608 GFP_KERNEL | __GFP_NOFAIL);
609 struct cpl_set_tcb_field *req;
610 u32 submode = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
611 + u32 pgcode = page_idx < ULP2_PGIDX_MAX ? page_idx : 0;
613 /* set up ulp submode and page size */
614 req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
615 @@ -339,8 +404,7 @@ int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *cconn, int hcrc, int dcrc)
617 req->word = htons(31);
618 req->mask = cpu_to_be64(0xFF000000);
619 - /* the connection page size is always the same as ddp-pgsz0 */
620 - req->val = cpu_to_be64(submode << 24);
621 + req->val = cpu_to_be64(((pgcode << 4) | submode) << 24);
622 skb->priority = CPL_PRIORITY_CONTROL;
624 cxgb3_ofld_send(c3cn->cdev, skb);
625 @@ -397,13 +461,15 @@ static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn,
626 segment->status = (conn->datadgst_en &&
627 (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) ?
628 ISCSI_SEGMENT_DGST_ERR : 0;
629 - if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
630 - cxgb3i_ddp_debug("opcode 0x%x, data %u, ddp'ed.\n",
631 - hdr->opcode & ISCSI_OPCODE_MASK,
632 - tcp_conn->in.datalen);
633 + if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED)
634 segment->total_copied = segment->total_size;
637 + cxgb3i_ddp_debug("opcode 0x%x, data %u, NOT ddp'ed, "
639 + hdr->opcode & ISCSI_OPCODE_MASK,
640 + tcp_conn->in.datalen, hdr->itt);
641 offset += sizeof(struct cpl_iscsi_hdr_norss);
644 while (segment->total_copied < segment->total_size) {
645 iscsi_tcp_segment_map(segment, 1);
646 @@ -481,67 +547,64 @@ int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *conn)
648 memset(dst, 0, padlen);
650 - unsigned int offset = 0;
652 - struct page *page = alloc_page(GFP_ATOMIC);
653 - int idx = skb_shinfo(skb)->nr_frags;
654 - skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
660 - frag->page_offset = 0;
661 - if (datalen > PAGE_SIZE)
662 - frag->size = PAGE_SIZE;
664 - frag->size = datalen;
665 - memcpy(page_address(page),
666 - data_seg->data + offset, frag->size);
668 - skb_shinfo(skb)->nr_frags++;
669 - datalen -= frag->size;
670 - offset += frag->size;
672 + struct page *pg = virt_to_page(data_seg->data);
675 + skb_fill_page_desc(skb, 0, pg,
676 + offset_in_page(data_seg->data),
678 + skb->len += datalen;
679 + skb->data_len += datalen;
680 + skb->truesize += datalen;
683 struct scatterlist *sg = data_seg->sg;
684 unsigned int offset = data_seg->sg_offset;
686 - int idx = skb_shinfo(skb)->nr_frags;
687 - skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
688 - struct page *pg = sg_page(sg);
692 - frag->page_offset = offset + sg->offset;
693 - frag->size = min(sg->length, datalen);
696 - skb_shinfo(skb)->nr_frags++;
697 - datalen -= frag->size;
700 + struct page *page = sg_page(sg);
701 + unsigned int sglen = sg->length - offset;
704 + int i = skb_shinfo(skb)->nr_frags;
709 + page = sg_page(sg);
711 + sglen = sg->length;
713 + copy = min(sglen, datalen);
715 + if (i && skb_can_coalesce(skb, i, page,
716 + sg->offset + offset)) {
717 + skb_shinfo(skb)->frags[i - 1].size += copy;
720 + skb_fill_page_desc(skb, i, page,
721 + sg->offset + offset, copy);
724 + skb->data_len += copy;
725 + skb->truesize += copy;
732 - if (skb_shinfo(skb)->nr_frags) {
734 - int idx = skb_shinfo(skb)->nr_frags;
735 - skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
736 - frag->page = pad_page;
737 - frag->page_offset = 0;
738 - frag->size = padlen;
739 - skb_shinfo(skb)->nr_frags++;
741 - datalen = data_seg->total_size + padlen;
742 - skb->data_len += datalen;
743 - skb->truesize += datalen;
744 - skb->len += datalen;
745 + if (padlen && skb_shinfo(skb)->nr_frags) {
746 + int idx = skb_shinfo(skb)->nr_frags;
747 + get_page(pad_page);
748 + skb_fill_page_desc(skb, idx, pad_page, 0, padlen);
749 + skb->data_len += padlen;
750 + skb->truesize += padlen;
751 + skb->len += padlen;
755 err = cxgb3i_c3cn_send_pdus((struct s3_conn *)tcp_conn->sock,
756 skb, MSG_DONTWAIT | MSG_NOSIGNAL);
759 int pdulen = hdrlen + datalen + padlen;
760 if (conn->hdrdgst_en)
761 @@ -556,7 +619,6 @@ send_pdu:
767 if (err < 0 && err != -EAGAIN) {
768 cxgb3i_log_error("conn 0x%p, xmit err %d.\n", conn, err);
769 @@ -649,13 +711,11 @@ void cxgb3i_conn_closing(struct s3_conn *c3cn)
770 int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
772 struct t3cdev *tdev = snic->tdev;
773 - struct cxgb3i_ddp_info *ddp = &snic->ddp;
774 + struct cxgb3i_ddp_info *ddp;
775 struct ulp_iscsi_info uinfo;
776 unsigned int ppmax, bits, max_bits;
779 - spin_lock_init(&ddp->map_lock);
781 err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
783 cxgb3i_log_error("%s, failed to get iscsi param err=%d.\n",
784 @@ -684,12 +744,21 @@ int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
785 snic->tag_format.rsvd_tag_mask =
786 (1 << (snic->tag_format.rsvd_bits + PPOD_IDX_SHIFT)) - 1;
788 - ddp->map = cxgb3i_alloc_big_mem(ppmax);
790 - cxgb3i_log_warn("snic unable to alloc ddp ppod 0x%u, "
791 - "ddp disabled.\n", ppmax);
792 + ddp = cxgb3i_alloc_big_mem(sizeof(struct cxgb3i_ddp_info) +
794 + (sizeof(struct cxgb3i_gather_list *) +
795 + sizeof(struct sk_buff *)));
797 + cxgb3i_log_warn("snic %s unable to alloc ddp ppod 0x%u, "
798 + "ddp disabled.\n", tdev->name, ppmax);
801 + ddp->gl_map = (struct cxgb3i_gather_list **)(ddp + 1);
802 + ddp->gl_skb = (struct sk_buff **)(((char *)ddp->gl_map) +
804 + sizeof(struct cxgb3i_gather_list *));
806 + spin_lock_init(&ddp->map_lock);
807 ddp->llimit = uinfo.llimit;
808 ddp->ulimit = uinfo.ulimit;
810 @@ -710,7 +779,7 @@ int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
812 ddp->idx_last = ppmax;
814 - tdev->ulp_iscsi = ddp;
815 + tdev->ulp_iscsi = snic->ddp = ddp;
817 cxgb3i_log_info("snic nppods %u (0x%x ~ 0x%x), rsvd shift %u, "
818 "bits %u, mask 0x%x, 0x%x, pkt %u,%u.\n",
819 @@ -723,19 +792,33 @@ int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
823 - cxgb3i_free_big_mem(ddp->map);
824 + cxgb3i_free_big_mem(ddp);
828 void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *snic)
830 - u8 *map = snic->ddp.map;
831 + struct cxgb3i_ddp_info *ddp = snic->ddp;
837 snic->tdev->ulp_iscsi = NULL;
838 spin_lock(&snic->lock);
839 - snic->ddp.map = NULL;
841 spin_unlock(&snic->lock);
842 - cxgb3i_free_big_mem(map);
844 + while (i < ddp->nppods) {
845 + struct cxgb3i_gather_list *gl = ddp->gl_map[i];
847 + int npods = (gl->nelem + PPOD_PAGES_MAX - 1)
848 + >> PPOD_PAGES_SHIFT;
851 + ddp_free_gl_skb(ddp, i, npods);
855 + cxgb3i_free_big_mem(ddp);