src/patches/suse-2.6.27.31/patches.drivers/cxgb3i-fixed-offload-array-size

   1 Subject: cxgb3i - fixed offload wr array size
   2 From: Karen Xie <kxie@chelsio.com>
   3 References: bnc#447409
   4
   5 On systems with 64K page size, the size of the wr request array used for calculating # of wrs required per pdu could be too small when scsi buffers are scattered on sector (512 bytes) boundary. This could cause the transmit to stop.
   6
   7 This patch changed the wr array size to be based on the sector size (512 bytes) instead of page size. It also added logic to coalesce scsi buffers as much as possible when constructing data pdus.
   8
   9 Signed-off-by: Karen Xie <kxie@chelsio.com>
  10 Signed-off-by: Chandra Seetharaman <sekharan@us.ibm.com>
  11 Signed-off-by: Hannes Reinecke <hare@suse.de>
  12 ---
  13 diff --git a/drivers/scsi/cxgb3i/cxgb3i.h b/drivers/scsi/cxgb3i/cxgb3i.h
  14 index e3f591f..48bc2ed 100644
  15 --- a/drivers/scsi/cxgb3i/cxgb3i.h
  16 +++ b/drivers/scsi/cxgb3i/cxgb3i.h
  17 @@ -62,6 +62,25 @@ struct cxgb3i_tag_format {
  18  };
  19
  20  /**
  21 + * struct cxgb3i_gather_list - cxgb3i direct data placement memory
  22 + *
  23 + * @tag:       ddp tag
  24 + * @length:    total data buffer length
  25 + * @offset:    initial offset to the 1st page
  26 + * @nelem:     # of pages
  27 + * @pages:     pages
  28 + * @phys_addr: physical address
  29 + */
  30 +struct cxgb3i_gather_list {
  31 +       u32 tag;
  32 +       unsigned int length;
  33 +       unsigned int offset;
  34 +       unsigned int nelem;
  35 +       struct page **pages;
  36 +       dma_addr_t phys_addr[0];
  37 +};
  38 +
  39 +/**
  40   * struct cxgb3i_ddp_info - cxgb3i direct data placement for pdu payload
  41   *
  42   * @llimit:    lower bound of the page pod memory
  43 @@ -77,7 +96,8 @@ struct cxgb3i_ddp_info {
  44         unsigned int nppods;
  45         unsigned int idx_last;
  46         spinlock_t map_lock;
  47 -       u8 *map;
  48 +       struct cxgb3i_gather_list **gl_map;
  49 +       struct sk_buff **gl_skb;
  50  };
  51
  52  /**
  53 @@ -119,7 +139,7 @@ struct cxgb3i_adapter {
  54         unsigned int rx_max_size;
  55
  56         struct cxgb3i_tag_format tag_format;
  57 -       struct cxgb3i_ddp_info ddp;
  58 +       struct cxgb3i_ddp_info *ddp;
  59  };
  60
  61  /**
  62 @@ -174,6 +194,6 @@ void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32,
  63                             struct scatterlist *, unsigned int);
  64  u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *, unsigned int,
  65                            u32, unsigned int, struct scatterlist *,
  66 -                          unsigned int);
  67 +                          unsigned int, int);
  68  int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *);
  69  #endif
  70 diff --git a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
  71 index 768c872..bc75dff 100644
  72 --- a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
  73 +++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
  74 @@ -723,14 +723,16 @@ static int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
  75                 struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock);
  76                 tag =
  77                     cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag,
  78 -                                          scsi_out(sc)->length,
  79 -                                          scsi_out(sc)->table.sgl,
  80 -                                          scsi_out(sc)->table.nents);
  81 +                                          scsi_in(sc)->length,
  82 +                                          scsi_in(sc)->table.sgl,
  83 +                                          scsi_in(sc)->table.nents,
  84 +                                          GFP_ATOMIC);
  85         }
  86         if (tag == RESERVED_ITT)
  87                 tag = sw_tag | (snic->tag_format.rsvd_mask <<
  88                                 snic->tag_format.rsvd_shift);
  89         *hdr_itt = htonl(tag);
  90 +
  91         return 0;
  92  }
  93
  94 @@ -744,8 +746,8 @@ static void cxgb3i_release_itt(struct iscsi_task *task, itt_t hdr_itt)
  95         hdr_itt = ntohl(hdr_itt);
  96         if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE))
  97                 cxgb3i_ddp_tag_release(snic, hdr_itt,
  98 -                                      scsi_out(sc)->table.sgl,
  99 -                                      scsi_out(sc)->table.nents);
 100 +                                      scsi_in(sc)->table.sgl,
 101 +                                      scsi_in(sc)->table.nents);
 102  }
 103
 104  /**
 105 diff --git a/drivers/scsi/cxgb3i/cxgb3i_offload.c b/drivers/scsi/cxgb3i/cxgb3i_offload.c
 106 index 8d0b3bf..e3f3008 100644
 107 --- a/drivers/scsi/cxgb3i/cxgb3i_offload.c
 108 +++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
 109 @@ -211,7 +211,8 @@ static unsigned int wrlen __read_mostly;
 110   * in the skb and whether it has any payload in its main body.  This maps the
 111   * length of the gather list represented by an skb into the # of necessary WRs.
 112   */
 113 -static unsigned int skb_wrs[MAX_SKB_FRAGS + 2] __read_mostly;
 114 +#define SKB_WR_LIST_SIZE       (16384/512 + 1)
 115 +static unsigned int skb_wrs[SKB_WR_LIST_SIZE + 2] __read_mostly;
 116
 117  static void s3_init_wr_tab(unsigned int wr_len)
 118  {
 119 @@ -220,7 +221,7 @@ static void s3_init_wr_tab(unsigned int wr_len)
 120         if (skb_wrs[1])         /* already initialized */
 121                 return;
 122
 123 -       for (i = 1; i < ARRAY_SIZE(skb_wrs); i++) {
 124 +       for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
 125                 int sgl_len = (3 * i) / 2 + (i & 1);
 126
 127                 sgl_len += 3;
 128 @@ -582,7 +583,6 @@ static inline void make_tx_data_wr(struct s3_conn *c3cn,
 129                            V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
 130
 131         if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
 132 -
 133                 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
 134                                     V_TX_CPU_IDX(c3cn->qset));
 135
 136 @@ -825,7 +825,7 @@ static int s3_push_frames(struct s3_conn *c3cn, int req_completion)
 137                 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
 138                         wrs_needed = 1;
 139
 140 -               WARN_ON(frags >= ARRAY_SIZE(skb_wrs) || wrs_needed < 1);
 141 +               WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
 142
 143                 if (c3cn->wr_avail < wrs_needed)
 144                         break;
 145 @@ -941,12 +941,23 @@ int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb, int flags)
 146                 goto out_err;
 147
 148         while (skb) {
 149 +               int frags = skb_shinfo(skb)->nr_frags +
 150 +                           (skb->len != skb->data_len);
 151 +
 152                 if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
 153                         c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
 154                         err = -EINVAL;
 155                         goto out_err;
 156                 }
 157
 158 +               if (frags >= SKB_WR_LIST_SIZE) {
 159 +                       cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
 160 +                                        c3cn, skb_shinfo(skb)->nr_frags,
 161 +                                        skb->len, skb->data_len);
 162 +                       err = -EINVAL;
 163 +                       goto out_err;
 164 +               }
 165 +
 166                 next = skb->next;
 167                 skb->next = NULL;
 168                 skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
 169 diff --git a/drivers/scsi/cxgb3i/cxgb3i_ulp2.c b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
 170 index 7b93ba0..fc1d413 100644
 171 --- a/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
 172 +++ b/drivers/scsi/cxgb3i/cxgb3i_ulp2.c
 173 @@ -45,36 +45,30 @@
 174  static struct page *pad_page;
 175
 176  #define ULP2_PGIDX_MAX         4
 177 -#define ULP2_4K_PAGE_SHIFT     12
 178 -#define ULP2_4K_PAGE_MASK      (~((1UL << ULP2_4K_PAGE_SHIFT) - 1))
 179 -static unsigned char ddp_page_order[ULP2_PGIDX_MAX];
 180 -static unsigned long ddp_page_size[ULP2_PGIDX_MAX];
 181 -static unsigned char ddp_page_shift[ULP2_PGIDX_MAX];
 182 +#define ULP2_DDP_THRESHOLD     2048
 183 +static unsigned char ddp_page_order[ULP2_PGIDX_MAX] = {0, 1, 2, 4};
 184 +static unsigned char ddp_page_shift[ULP2_PGIDX_MAX] = {12, 13, 14, 16};
 185  static unsigned char sw_tag_idx_bits;
 186  static unsigned char sw_tag_age_bits;
 187 +static unsigned char page_idx = ULP2_PGIDX_MAX;
 188
 189  static void cxgb3i_ddp_page_init(void)
 190  {
 191         int i;
 192 -       unsigned long n = PAGE_SIZE >> ULP2_4K_PAGE_SHIFT;
 193 -
 194 -       if (PAGE_SIZE & (~ULP2_4K_PAGE_MASK)) {
 195 -               cxgb3i_log_debug("PAGE_SIZE 0x%lx is not multiple of 4K, "
 196 -                               "ddp disabled.\n", PAGE_SIZE);
 197 -               return;
 198 -       }
 199 -       n = __ilog2_u32(n);
 200 -       for (i = 0; i < ULP2_PGIDX_MAX; i++, n++) {
 201 -               ddp_page_order[i] = n;
 202 -               ddp_page_shift[i] = ULP2_4K_PAGE_SHIFT + n;
 203 -               ddp_page_size[i] = 1 << ddp_page_shift[i];
 204 -               cxgb3i_log_debug("%d, order %u, shift %u, size 0x%lx.\n", i,
 205 -                                ddp_page_order[i], ddp_page_shift[i],
 206 -                                ddp_page_size[i]);
 207 -       }
 208
 209         sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
 210         sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
 211 +
 212 +       for (i = 0; i < ULP2_PGIDX_MAX; i++) {
 213 +               if (PAGE_SIZE == (1UL << ddp_page_shift[i])) {
 214 +                       page_idx = i;
 215 +                       cxgb3i_log_info("PAGE_SIZE %lu, idx %u.\n",
 216 +                                       PAGE_SIZE, page_idx);
 217 +               }
 218 +       }
 219 +
 220 +       if (page_idx == ULP2_PGIDX_MAX)
 221 +               cxgb3i_log_info("PAGE_SIZE %lu, no match.\n", PAGE_SIZE);
 222  }
 223
 224  static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
 225 @@ -91,38 +85,27 @@ static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
 226
 227  static int set_ddp_map(struct cxgb3i_adapter *snic, struct pagepod_hdr *hdr,
 228                        unsigned int idx, unsigned int npods,
 229 -                      struct scatterlist *sgl, unsigned int sgcnt)
 230 +                      struct cxgb3i_gather_list *gl)
 231  {
 232 -       struct cxgb3i_ddp_info *ddp = &snic->ddp;
 233 -       struct scatterlist *sg = sgl;
 234 +       struct cxgb3i_ddp_info *ddp = snic->ddp;
 235         unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
 236         int i;
 237
 238 -       for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
 239 -               struct sk_buff *skb;
 240 +       for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
 241 +               struct sk_buff *skb = ddp->gl_skb[idx];
 242                 struct pagepod *ppod;
 243 -               int j, k;
 244 -               skb =
 245 -                   alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
 246 -                             GFP_ATOMIC);
 247 -               if (!skb) {
 248 -                       cxgb3i_log_debug("skb OMM.\n");
 249 -                       return -ENOMEM;
 250 -               }
 251 -               skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
 252 +               int j, pidx;
 253 +
 254 +               /* hold on to the skb until we clear the ddp mapping */
 255 +               skb_get(skb);
 256
 257                 ulp_mem_io_set_hdr(skb, pm_addr);
 258                 ppod =
 259                     (struct pagepod *)(skb->head + sizeof(struct ulp_mem_io));
 260                 memcpy(&(ppod->hdr), hdr, sizeof(struct pagepod));
 261 -               for (j = 0, k = i * 4; j < 5; j++, k++) {
 262 -                       if (k < sgcnt) {
 263 -                               ppod->addr[j] = cpu_to_be64(sg_dma_address(sg));
 264 -                               if (j < 4)
 265 -                                       sg = sg_next(sg);
 266 -                       } else
 267 -                               ppod->addr[j] = 0UL;
 268 -               }
 269 +               for (pidx = 4 * i, j = 0; j < 5; ++j, ++pidx)
 270 +                       ppod->addr[j] = pidx < gl->nelem ?
 271 +                                    cpu_to_be64(gl->phys_addr[pidx]) : 0UL;
 272
 273                 skb->priority = CPL_PRIORITY_CONTROL;
 274                 cxgb3_ofld_send(snic->tdev, skb);
 275 @@ -133,18 +116,14 @@ static int set_ddp_map(struct cxgb3i_adapter *snic, struct pagepod_hdr *hdr,
 276  static int clear_ddp_map(struct cxgb3i_adapter *snic, unsigned int idx,
 277                          unsigned int npods)
 278  {
 279 -       struct cxgb3i_ddp_info *ddp = &snic->ddp;
 280 +       struct cxgb3i_ddp_info *ddp = snic->ddp;
 281         unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
 282         int i;
 283
 284 -       for (i = 0; i < npods; i++, pm_addr += PPOD_SIZE) {
 285 -               struct sk_buff *skb;
 286 -               skb =
 287 -                   alloc_skb(sizeof(struct ulp_mem_io) + PPOD_SIZE,
 288 -                             GFP_ATOMIC);
 289 -               if (!skb)
 290 -                       return -ENOMEM;
 291 -               skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
 292 +       for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
 293 +               struct sk_buff *skb = ddp->gl_skb[idx];
 294 +
 295 +               ddp->gl_skb[idx] = NULL;
 296                 memset((skb->head + sizeof(struct ulp_mem_io)), 0, PPOD_SIZE);
 297                 ulp_mem_io_set_hdr(skb, pm_addr);
 298                 skb->priority = CPL_PRIORITY_CONTROL;
 299 @@ -153,42 +132,80 @@ static int clear_ddp_map(struct cxgb3i_adapter *snic, unsigned int idx,
 300         return 0;
 301  }
 302
 303 -static int cxgb3i_ddp_sgl_check(struct scatterlist *sgl, unsigned int sgcnt)
 304 +static struct cxgb3i_gather_list *ddp_make_gl(unsigned int xferlen,
 305 +                                             struct scatterlist *sgl,
 306 +                                             unsigned int sgcnt, int gfp)
 307  {
 308 -       struct scatterlist *sg;
 309 -       int i;
 310 -
 311 -       /* make sure the sgl is fit for ddp:
 312 -        *      each has the same page size, and
 313 -        *      first & last page do not need to be used completely, and
 314 -        *      the rest of page must be used completely
 315 -        */
 316 -       for_each_sg(sgl, sg, sgcnt, i) {
 317 -               if ((i && sg->offset) ||
 318 -                   ((i != sgcnt - 1) &&
 319 -                    (sg->length + sg->offset) != PAGE_SIZE)) {
 320 -                       cxgb3i_tag_debug("sg %u/%u, off %u, len %u.\n",
 321 -                                        i, sgcnt, sg->offset, sg->length);
 322 -                       return -EINVAL;
 323 +       struct cxgb3i_gather_list *gl;
 324 +       struct scatterlist *sg = sgl;
 325 +       struct page *sgpage = sg_page(sg);
 326 +       unsigned int sglen = sg->length;
 327 +       unsigned int sgoffset = sg->offset;
 328 +       unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >>
 329 +                             PAGE_SHIFT;
 330 +       int i = 1, j = 0;
 331 +
 332 +       gl = kzalloc(sizeof(struct cxgb3i_gather_list) +
 333 +                    npages * (sizeof(dma_addr_t) + sizeof(struct page *)),
 334 +                    gfp);
 335 +       if (!gl)
 336 +               return NULL;
 337 +
 338 +       gl->pages = (struct page **)&gl->phys_addr[npages];
 339 +       gl->length = xferlen;
 340 +       gl->offset = sgoffset;
 341 +       gl->pages[0] = sgpage;
 342 +
 343 +       sg = sg_next(sg);
 344 +       while (sg) {
 345 +               struct page *page = sg_page(sg);
 346 +
 347 +               if (sgpage == page && sg->offset == sgoffset + sglen)
 348 +                       sglen += sg->length;
 349 +               else {
 350 +                       /* make sure the sgl is fit for ddp:
 351 +                        * each has the same page size, and
 352 +                        * all of the middle pages are used completely
 353 +                        */
 354 +                       if ((j && sgoffset) ||
 355 +                           ((i != sgcnt - 1) &&
 356 +                            ((sglen + sgoffset) & ~PAGE_MASK)))
 357 +                               goto error_out;
 358 +
 359 +                       j++;
 360 +                       if (j == gl->nelem)
 361 +                               goto error_out;
 362 +                       gl->pages[j] = page;
 363 +                       sglen = sg->length;
 364 +                       sgoffset = sg->offset;
 365 +                       sgpage = page;
 366                 }
 367 +               i++;
 368 +               sg = sg_next(sg);
 369         }
 370 +       gl->nelem = ++j;
 371 +       return gl;
 372
 373 -       return 0;
 374 +error_out:
 375 +       kfree(gl);
 376 +       return NULL;
 377  }
 378
 379  static inline int ddp_find_unused_entries(struct cxgb3i_ddp_info *ddp,
 380 -                                         int start, int max, int count)
 381 +                                         int start, int max, int count,
 382 +                                         struct cxgb3i_gather_list *gl)
 383  {
 384         unsigned int i, j;
 385
 386         spin_lock(&ddp->map_lock);
 387         for (i = start; i <= max;) {
 388                 for (j = 0; j < count; j++) {
 389 -                       if (ddp->map[i + j])
 390 +                       if (ddp->gl_map[i + j])
 391                                 break;
 392                 }
 393                 if (j == count) {
 394 -                       memset(&ddp->map[i], 1, count);
 395 +                       for (j = 0; j < count; j++)
 396 +                               ddp->gl_map[i + j] = gl;
 397                         spin_unlock(&ddp->map_lock);
 398                         return i;
 399                 }
 400 @@ -202,86 +219,127 @@ static inline void ddp_unmark_entries(struct cxgb3i_ddp_info *ddp,
 401                                       int start, int count)
 402  {
 403         spin_lock(&ddp->map_lock);
 404 -       memset(&ddp->map[start], 0, count);
 405 +       memset(&ddp->gl_map[start], 0,
 406 +              count * sizeof(struct cxgb3i_gather_list *));
 407         spin_unlock(&ddp->map_lock);
 408  }
 409
 410 -static inline int sgl_map(struct cxgb3i_adapter *snic,
 411 -                         struct scatterlist *sgl, unsigned int sgcnt)
 412 +static inline void ddp_free_gl_skb(struct cxgb3i_ddp_info *ddp,
 413 +                                  int idx, int count)
 414  {
 415 -       struct scatterlist *sg;
 416 -       int i, err;
 417 +       int i;
 418
 419 -       for_each_sg(sgl, sg, sgcnt, i) {
 420 -               err = pci_map_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
 421 -               if (err <= 0) {
 422 -                       cxgb3i_tag_debug("sgcnt %d/%u, pci map failed %d.\n",
 423 -                                        i, sgcnt, err);
 424 -                       return err;
 425 +       for (i = 0; i < count; i++, idx++)
 426 +               if (ddp->gl_skb[idx]) {
 427 +                       kfree_skb(ddp->gl_skb[idx]);
 428 +                       ddp->gl_skb[idx] = NULL;
 429 +               }
 430 +}
 431 +
 432 +static inline int ddp_alloc_gl_skb(struct cxgb3i_ddp_info *ddp,
 433 +                                  int idx, int count, int gfp)
 434 +{
 435 +       int i;
 436 +
 437 +       for (i = 0; i < count; i++) {
 438 +               struct sk_buff *skb = alloc_skb(sizeof(struct ulp_mem_io) +
 439 +                                               PPOD_SIZE, gfp);
 440 +               if (skb) {
 441 +                       ddp->gl_skb[idx + i] = skb;
 442 +                       skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
 443 +               } else {
 444 +                       ddp_free_gl_skb(ddp, idx, i);
 445 +                       return -ENOMEM;
 446                 }
 447         }
 448 -       return sgcnt;
 449 +       return 0;
 450  }
 451
 452 -static inline void sgl_unmap(struct cxgb3i_adapter *snic,
 453 -                            struct scatterlist *sgl, unsigned int sgcnt)
 454 +static inline void ddp_gl_unmap(struct pci_dev *pdev,
 455 +                               struct cxgb3i_gather_list *gl)
 456  {
 457 -       struct scatterlist *sg;
 458         int i;
 459
 460 -       for_each_sg(sgl, sg, sgcnt, i) {
 461 -               if (sg_dma_address(sg))
 462 -                       pci_unmap_sg(snic->pdev, sg, 1, PCI_DMA_FROMDEVICE);
 463 -               else
 464 -                       break;
 465 +       for (i = 0; i < gl->nelem; i++)
 466 +               pci_unmap_page(pdev, gl->phys_addr[i], PAGE_SIZE,
 467 +                              PCI_DMA_FROMDEVICE);
 468 +}
 469 +
 470 +static inline int ddp_gl_map(struct pci_dev *pdev,
 471 +                            struct cxgb3i_gather_list *gl)
 472 +{
 473 +       int i;
 474 +
 475 +       for (i = 0; i < gl->nelem; i++) {
 476 +               gl->phys_addr[i] = pci_map_page(pdev, gl->pages[i], 0,
 477 +                                               PAGE_SIZE,
 478 +                                               PCI_DMA_FROMDEVICE);
 479 +               if (unlikely(pci_dma_mapping_error(pdev, gl->phys_addr[i])))
 480 +                       goto unmap;
 481         }
 482 +
 483 +       return i;
 484 +
 485 +unmap:
 486 +       if (i) {
 487 +               unsigned int nelem = gl->nelem;
 488 +
 489 +               gl->nelem = i;
 490 +               ddp_gl_unmap(pdev, gl);
 491 +               gl->nelem = nelem;
 492 +       }
 493 +       return -ENOMEM;
 494  }
 495
 496  u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *snic, unsigned int tid,
 497                            u32 sw_tag, unsigned int xferlen,
 498 -                          struct scatterlist *sgl, unsigned int sgcnt)
 499 +                          struct scatterlist *sgl, unsigned int sgcnt,
 500 +                          int gfp)
 501  {
 502 -       struct cxgb3i_ddp_info *ddp = &snic->ddp;
 503 +       struct cxgb3i_ddp_info *ddp = snic->ddp;
 504 +       struct cxgb3i_gather_list *gl;
 505         struct pagepod_hdr hdr;
 506         unsigned int npods;
 507         int idx = -1, idx_max;
 508         u32 tag;
 509 -       int err;
 510
 511 -       if (!ddp || !sgcnt || xferlen < PAGE_SIZE) {
 512 -               cxgb3i_tag_debug("sgcnt %u, xferlen %u < %lu, NO DDP.\n",
 513 -                                sgcnt, xferlen, PAGE_SIZE);
 514 +       if (page_idx || !ddp || !sgcnt || xferlen < ULP2_DDP_THRESHOLD) {
 515 +               cxgb3i_tag_debug("pgidx %u, sgcnt %u, xfer %u/%u, NO ddp.\n",
 516 +                                page_idx, sgcnt, xferlen, ULP2_DDP_THRESHOLD);
 517                 return RESERVED_ITT;
 518         }
 519
 520 -       err = cxgb3i_ddp_sgl_check(sgl, sgcnt);
 521 -       if (err < 0) {
 522 +       gl = ddp_make_gl(xferlen, sgl, sgcnt, gfp);
 523 +       if (!gl) {
 524                 cxgb3i_tag_debug("sgcnt %u, xferlen %u, SGL check fail.\n",
 525                                  sgcnt, xferlen);
 526                 return RESERVED_ITT;
 527         }
 528
 529 -       npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
 530 +       npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
 531         idx_max = ddp->nppods - npods + 1;
 532
 533         if (ddp->idx_last == ddp->nppods)
 534 -               idx = ddp_find_unused_entries(ddp, 0, idx_max, npods);
 535 +               idx = ddp_find_unused_entries(ddp, 0, idx_max, npods, gl);
 536         else {
 537                 idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1, idx_max,
 538 -                                             npods);
 539 +                                             npods, gl);
 540                 if ((idx < 0) && (ddp->idx_last >= npods))
 541                         idx = ddp_find_unused_entries(ddp, 0,
 542                                                       ddp->idx_last - npods + 1,
 543 -                                                     npods);
 544 +                                                     npods, gl);
 545         }
 546         if (idx < 0) {
 547 +               kfree(gl);
 548                 cxgb3i_tag_debug("sgcnt %u, xferlen %u, npods %u NO DDP.\n",
 549                                  sgcnt, xferlen, npods);
 550                 return RESERVED_ITT;
 551         }
 552
 553 -       err = sgl_map(snic, sgl, sgcnt);
 554 -       if (err < sgcnt)
 555 +       if (ddp_alloc_gl_skb(ddp, idx, npods, gfp) < 0)
 556 +               goto unmark_entries;
 557 +
 558 +       if (ddp_gl_map(snic->pdev, gl) < 0)
 559                 goto unmap_sgl;
 560
 561         tag = sw_tag | (idx << snic->tag_format.rsvd_shift);
 562 @@ -290,9 +348,9 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *snic, unsigned int tid,
 563         hdr.vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid));
 564         hdr.pgsz_tag_clr = htonl(tag & snic->tag_format.rsvd_tag_mask);
 565         hdr.maxoffset = htonl(xferlen);
 566 -       hdr.pgoffset = htonl(sgl->offset);
 567 +       hdr.pgoffset = htonl(gl->offset);
 568
 569 -       if (set_ddp_map(snic, &hdr, idx, npods, sgl, sgcnt) < 0)
 570 +       if (set_ddp_map(snic, &hdr, idx, npods, gl) < 0)
 571                 goto unmap_sgl;
 572
 573         ddp->idx_last = idx;
 574 @@ -301,7 +359,9 @@ u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *snic, unsigned int tid,
 575         return tag;
 576
 577  unmap_sgl:
 578 -       sgl_unmap(snic, sgl, sgcnt);
 579 +       ddp_gl_unmap(snic->pdev, gl);
 580 +       ddp_free_gl_skb(ddp, idx, npods);
 581 +unmark_entries:
 582         ddp_unmark_entries(ddp, idx, npods);
 583         return RESERVED_ITT;
 584  }
 585 @@ -311,14 +371,18 @@ void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *snic, u32 tag,
 586  {
 587         u32 idx = (tag >> snic->tag_format.rsvd_shift) &
 588                   snic->tag_format.rsvd_mask;
 589 -       unsigned int npods = (sgcnt + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
 590
 591         if (idx < snic->tag_format.rsvd_mask) {
 592 +               struct cxgb3i_ddp_info *ddp = snic->ddp;
 593 +               struct cxgb3i_gather_list *gl = ddp->gl_map[idx];
 594 +               unsigned int npods = (gl->nelem + PPOD_PAGES_MAX - 1) >>
 595 +                                    PPOD_PAGES_SHIFT;
 596 +
 597                 cxgb3i_tag_debug("ddp tag 0x%x, release idx 0x%x, npods %u.\n",
 598                                  tag, idx, npods);
 599                 clear_ddp_map(snic, idx, npods);
 600 -               ddp_unmark_entries(&snic->ddp, idx, npods);
 601 -               sgl_unmap(snic, sgl, sgcnt);
 602 +               ddp_unmark_entries(ddp, idx, npods);
 603 +               ddp_gl_unmap(snic->pdev, gl);
 604         }
 605  }
 606
 607 @@ -330,6 +394,7 @@ int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *cconn, int hcrc, int dcrc)
 608                                         GFP_KERNEL | __GFP_NOFAIL);
 609         struct cpl_set_tcb_field *req;
 610         u32 submode = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
 611 +       u32 pgcode = page_idx < ULP2_PGIDX_MAX ? page_idx : 0;
 612
 613         /* set up ulp submode and page size */
 614         req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
 615 @@ -339,8 +404,7 @@ int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *cconn, int hcrc, int dcrc)
 616         req->cpu_idx = 0;
 617         req->word = htons(31);
 618         req->mask = cpu_to_be64(0xFF000000);
 619 -       /* the connection page size is always the same as ddp-pgsz0 */
 620 -       req->val = cpu_to_be64(submode << 24);
 621 +       req->val = cpu_to_be64(((pgcode << 4) | submode) << 24);
 622         skb->priority = CPL_PRIORITY_CONTROL;
 623
 624         cxgb3_ofld_send(c3cn->cdev, skb);
 625 @@ -397,13 +461,15 @@ static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn,
 626                 segment->status = (conn->datadgst_en &&
 627                                    (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) ?
 628                     ISCSI_SEGMENT_DGST_ERR : 0;
 629 -               if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
 630 -                       cxgb3i_ddp_debug("opcode 0x%x, data %u, ddp'ed.\n",
 631 -                                        hdr->opcode & ISCSI_OPCODE_MASK,
 632 -                                        tcp_conn->in.datalen);
 633 +               if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED)
 634                         segment->total_copied = segment->total_size;
 635 -               } else
 636 +               else {
 637 +                       cxgb3i_ddp_debug("opcode 0x%x, data %u, NOT ddp'ed, "
 638 +                                        "itt 0x%x.\n",
 639 +                                        hdr->opcode & ISCSI_OPCODE_MASK,
 640 +                                        tcp_conn->in.datalen, hdr->itt);
 641                         offset += sizeof(struct cpl_iscsi_hdr_norss);
 642 +               }
 643
 644                 while (segment->total_copied < segment->total_size) {
 645                         iscsi_tcp_segment_map(segment, 1);
 646 @@ -481,67 +547,64 @@ int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *conn)
 647                         if (padlen)
 648                                 memset(dst, 0, padlen);
 649                 } else {
 650 -                       unsigned int offset = 0;
 651 -                       while (datalen) {
 652 -                               struct page *page = alloc_page(GFP_ATOMIC);
 653 -                               int idx = skb_shinfo(skb)->nr_frags;
 654 -                               skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
 655 -
 656 -                               if (!page)
 657 -                                       goto free_skb;
 658 -
 659 -                               frag->page = page;
 660 -                               frag->page_offset = 0;
 661 -                               if (datalen > PAGE_SIZE)
 662 -                                       frag->size = PAGE_SIZE;
 663 -                               else
 664 -                                       frag->size = datalen;
 665 -                               memcpy(page_address(page),
 666 -                                      data_seg->data + offset, frag->size);
 667 -
 668 -                               skb_shinfo(skb)->nr_frags++;
 669 -                               datalen -= frag->size;
 670 -                               offset += frag->size;
 671 -                       }
 672 +                       struct page *pg = virt_to_page(data_seg->data);
 673 +
 674 +                       get_page(pg);
 675 +                       skb_fill_page_desc(skb, 0, pg,
 676 +                                          offset_in_page(data_seg->data),
 677 +                                          datalen);
 678 +                       skb->len += datalen;
 679 +                       skb->data_len += datalen;
 680 +                       skb->truesize += datalen;
 681                 }
 682         } else {
 683                 struct scatterlist *sg = data_seg->sg;
 684                 unsigned int offset = data_seg->sg_offset;
 685 -               while (datalen) {
 686 -                       int idx = skb_shinfo(skb)->nr_frags;
 687 -                       skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
 688 -                       struct page *pg = sg_page(sg);
 689 -
 690 -                       get_page(pg);
 691 -                       frag->page = pg;
 692 -                       frag->page_offset = offset + sg->offset;
 693 -                       frag->size = min(sg->length, datalen);
 694 -
 695 -                       offset = 0;
 696 -                       skb_shinfo(skb)->nr_frags++;
 697 -                       datalen -= frag->size;
 698 -                       sg = sg_next(sg);
 699 -               }
 700 +               struct page *page = sg_page(sg);
 701 +               unsigned int sglen = sg->length - offset;
 702 +
 703 +               do {
 704 +                       int i = skb_shinfo(skb)->nr_frags;
 705 +                       unsigned int copy;
 706 +
 707 +                       if (!sglen) {
 708 +                               sg = sg_next(sg);
 709 +                               page = sg_page(sg);
 710 +                               offset = 0;
 711 +                               sglen = sg->length;
 712 +                       }
 713 +                       copy = min(sglen, datalen);
 714 +
 715 +                       if (i && skb_can_coalesce(skb, i, page,
 716 +                                                 sg->offset + offset)) {
 717 +                               skb_shinfo(skb)->frags[i - 1].size += copy;
 718 +                       } else {
 719 +                               get_page(page);
 720 +                               skb_fill_page_desc(skb, i, page,
 721 +                                                  sg->offset + offset, copy);
 722 +                       }
 723 +                       skb->len += copy;
 724 +                       skb->data_len += copy;
 725 +                       skb->truesize += copy;
 726 +                       offset += copy;
 727 +                       sglen -= copy;
 728 +                       datalen -= copy;
 729 +               } while (datalen);
 730         }
 731
 732 -       if (skb_shinfo(skb)->nr_frags) {
 733 -               if (padlen) {
 734 -                       int idx = skb_shinfo(skb)->nr_frags;
 735 -                       skb_frag_t *frag = &skb_shinfo(skb)->frags[idx];
 736 -                       frag->page = pad_page;
 737 -                       frag->page_offset = 0;
 738 -                       frag->size = padlen;
 739 -                       skb_shinfo(skb)->nr_frags++;
 740 -               }
 741 -               datalen = data_seg->total_size + padlen;
 742 -               skb->data_len += datalen;
 743 -               skb->truesize += datalen;
 744 -               skb->len += datalen;
 745 +       if (padlen && skb_shinfo(skb)->nr_frags) {
 746 +               int idx = skb_shinfo(skb)->nr_frags;
 747 +               get_page(pad_page);
 748 +               skb_fill_page_desc(skb, idx, pad_page, 0, padlen);
 749 +               skb->data_len += padlen;
 750 +               skb->truesize += padlen;
 751 +               skb->len += padlen;
 752         }
 753
 754  send_pdu:
 755         err = cxgb3i_c3cn_send_pdus((struct s3_conn *)tcp_conn->sock,
 756                                     skb, MSG_DONTWAIT | MSG_NOSIGNAL);
 757 +
 758         if (err > 0) {
 759                 int pdulen = hdrlen + datalen + padlen;
 760                 if (conn->hdrdgst_en)
 761 @@ -556,7 +619,6 @@ send_pdu:
 762                 return pdulen;
 763         }
 764
 765 -free_skb:
 766         kfree_skb(skb);
 767         if (err < 0 && err != -EAGAIN) {
 768                 cxgb3i_log_error("conn 0x%p, xmit err %d.\n", conn, err);
 769 @@ -649,13 +711,11 @@ void cxgb3i_conn_closing(struct s3_conn *c3cn)
 770  int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
 771  {
 772         struct t3cdev *tdev = snic->tdev;
 773 -       struct cxgb3i_ddp_info *ddp = &snic->ddp;
 774 +       struct cxgb3i_ddp_info *ddp;
 775         struct ulp_iscsi_info uinfo;
 776         unsigned int ppmax, bits, max_bits;
 777         int i, err;
 778
 779 -       spin_lock_init(&ddp->map_lock);
 780 -
 781         err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
 782         if (err < 0) {
 783                 cxgb3i_log_error("%s, failed to get iscsi param err=%d.\n",
 784 @@ -684,12 +744,21 @@ int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
 785         snic->tag_format.rsvd_tag_mask =
 786                 (1 << (snic->tag_format.rsvd_bits + PPOD_IDX_SHIFT)) - 1;
 787
 788 -       ddp->map = cxgb3i_alloc_big_mem(ppmax);
 789 -       if (!ddp->map) {
 790 -               cxgb3i_log_warn("snic unable to alloc ddp ppod 0x%u, "
 791 -                               "ddp disabled.\n", ppmax);
 792 +       ddp = cxgb3i_alloc_big_mem(sizeof(struct cxgb3i_ddp_info) +
 793 +                                  ppmax *
 794 +                                  (sizeof(struct cxgb3i_gather_list *) +
 795 +                                   sizeof(struct sk_buff *)));
 796 +       if (!ddp) {
 797 +               cxgb3i_log_warn("snic %s unable to alloc ddp ppod 0x%u, "
 798 +                               "ddp disabled.\n", tdev->name, ppmax);
 799                 return 0;
 800         }
 801 +       ddp->gl_map = (struct cxgb3i_gather_list **)(ddp + 1);
 802 +       ddp->gl_skb = (struct sk_buff **)(((char *)ddp->gl_map) +
 803 +                                         ppmax *
 804 +                                         sizeof(struct cxgb3i_gather_list *));
 805 +
 806 +       spin_lock_init(&ddp->map_lock);
 807         ddp->llimit = uinfo.llimit;
 808         ddp->ulimit = uinfo.ulimit;
 809
 810 @@ -710,7 +779,7 @@ int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
 811         ddp->nppods = ppmax;
 812         ddp->idx_last = ppmax;
 813
 814 -       tdev->ulp_iscsi = ddp;
 815 +       tdev->ulp_iscsi = snic->ddp = ddp;
 816
 817         cxgb3i_log_info("snic nppods %u (0x%x ~ 0x%x), rsvd shift %u, "
 818                         "bits %u, mask 0x%x, 0x%x, pkt %u,%u.\n",
 819 @@ -723,19 +792,33 @@ int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *snic)
 820         return 0;
 821
 822  free_ppod_map:
 823 -       cxgb3i_free_big_mem(ddp->map);
 824 +       cxgb3i_free_big_mem(ddp);
 825         return 0;
 826  }
 827
 828  void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *snic)
 829  {
 830 -       u8 *map = snic->ddp.map;
 831 +       struct cxgb3i_ddp_info *ddp = snic->ddp;
 832 +
 833 +       if (ddp) {
 834 +               int i = 0;
 835
 836 -       if (map) {
 837                 snic->tdev->ulp_iscsi = NULL;
 838                 spin_lock(&snic->lock);
 839 -               snic->ddp.map = NULL;
 840 +               snic->ddp = NULL;
 841                 spin_unlock(&snic->lock);
 842 -               cxgb3i_free_big_mem(map);
 843 +
 844 +               while (i < ddp->nppods) {
 845 +                       struct cxgb3i_gather_list *gl = ddp->gl_map[i];
 846 +                       if (gl) {
 847 +                               int npods = (gl->nelem + PPOD_PAGES_MAX - 1)
 848 +                                            >> PPOD_PAGES_SHIFT;
 849 +
 850 +                               kfree(gl);
 851 +                               ddp_free_gl_skb(ddp, i, npods);
 852 +                       } else
 853 +                               i++;
 854 +               }
 855 +               cxgb3i_free_big_mem(ddp);
 856         }
 857  }