drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2016-2017 Broadcom Limited
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  */
   9 #include <linux/kernel.h>
  10 #include <linux/errno.h>
  11 #include <linux/pci.h>
  12 #include <linux/netdevice.h>
  13 #include <linux/etherdevice.h>
  14 #include <linux/if_vlan.h>
  15 #include <linux/bpf.h>
  16 #include <linux/bpf_trace.h>
  17 #include <linux/filter.h>
  18 #include <net/page_pool.h>
  19 #include "bnxt_hsi.h"
  20 #include "bnxt.h"
  21 #include "bnxt_xdp.h"
  22
  23 DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);
  24
  25 struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
  26                                    struct bnxt_tx_ring_info *txr,
  27                                    dma_addr_t mapping, u32 len,
  28                                    struct xdp_buff *xdp)
  29 {
  30         struct skb_shared_info *sinfo;
  31         struct bnxt_sw_tx_bd *tx_buf;
  32         struct tx_bd *txbd;
  33         int num_frags = 0;
  34         u32 flags;
  35         u16 prod;
  36         int i;
  37
  38         if (xdp && xdp_buff_has_frags(xdp)) {
  39                 sinfo = xdp_get_shared_info_from_buff(xdp);
  40                 num_frags = sinfo->nr_frags;
  41         }
  42
  43         /* fill up the first buffer */
  44         prod = txr->tx_prod;
  45         tx_buf = &txr->tx_buf_ring[prod];
  46         tx_buf->nr_frags = num_frags;
  47         if (xdp)
  48                 tx_buf->page = virt_to_head_page(xdp->data);
  49
  50         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
  51         flags = (len << TX_BD_LEN_SHIFT) |
  52                 ((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT) |
  53                 bnxt_lhint_arr[len >> 9];
  54         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
  55         txbd->tx_bd_opaque = prod;
  56         txbd->tx_bd_haddr = cpu_to_le64(mapping);
  57
  58         /* now let us fill up the frags into the next buffers */
  59         for (i = 0; i < num_frags ; i++) {
  60                 skb_frag_t *frag = &sinfo->frags[i];
  61                 struct bnxt_sw_tx_bd *frag_tx_buf;
  62                 struct pci_dev *pdev = bp->pdev;
  63                 dma_addr_t frag_mapping;
  64                 int frag_len;
  65
  66                 prod = NEXT_TX(prod);
  67                 txr->tx_prod = prod;
  68
  69                 /* first fill up the first buffer */
  70                 frag_tx_buf = &txr->tx_buf_ring[prod];
  71                 frag_tx_buf->page = skb_frag_page(frag);
  72
  73                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
  74
  75                 frag_len = skb_frag_size(frag);
  76                 frag_mapping = skb_frag_dma_map(&pdev->dev, frag, 0,
  77                                                 frag_len, DMA_TO_DEVICE);
  78
  79                 if (unlikely(dma_mapping_error(&pdev->dev, frag_mapping)))
  80                         return NULL;
  81
  82                 dma_unmap_addr_set(frag_tx_buf, mapping, frag_mapping);
  83
  84                 flags = frag_len << TX_BD_LEN_SHIFT;
  85                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
  86                 txbd->tx_bd_haddr = cpu_to_le64(frag_mapping);
  87
  88                 len = frag_len;
  89         }
  90
  91         flags &= ~TX_BD_LEN;
  92         txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags |
  93                         TX_BD_FLAGS_PACKET_END);
  94         /* Sync TX BD */
  95         wmb();
  96         prod = NEXT_TX(prod);
  97         txr->tx_prod = prod;
  98
  99         return tx_buf;
 100 }
 101
 102 static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 103                             dma_addr_t mapping, u32 len, u16 rx_prod,
 104                             struct xdp_buff *xdp)
 105 {
 106         struct bnxt_sw_tx_bd *tx_buf;
 107
 108         tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp);
 109         tx_buf->rx_prod = rx_prod;
 110         tx_buf->action = XDP_TX;
 111
 112 }
 113
 114 static void __bnxt_xmit_xdp_redirect(struct bnxt *bp,
 115                                      struct bnxt_tx_ring_info *txr,
 116                                      dma_addr_t mapping, u32 len,
 117                                      struct xdp_frame *xdpf)
 118 {
 119         struct bnxt_sw_tx_bd *tx_buf;
 120
 121         tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL);
 122         tx_buf->action = XDP_REDIRECT;
 123         tx_buf->xdpf = xdpf;
 124         dma_unmap_addr_set(tx_buf, mapping, mapping);
 125         dma_unmap_len_set(tx_buf, len, 0);
 126 }
 127
 128 void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 129 {
 130         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 131         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 132         bool rx_doorbell_needed = false;
 133         struct bnxt_sw_tx_bd *tx_buf;
 134         u16 tx_cons = txr->tx_cons;
 135         u16 last_tx_cons = tx_cons;
 136         int i, j, frags;
 137
 138         for (i = 0; i < nr_pkts; i++) {
 139                 tx_buf = &txr->tx_buf_ring[tx_cons];
 140
 141                 if (tx_buf->action == XDP_REDIRECT) {
 142                         struct pci_dev *pdev = bp->pdev;
 143
 144                         dma_unmap_single(&pdev->dev,
 145                                          dma_unmap_addr(tx_buf, mapping),
 146                                          dma_unmap_len(tx_buf, len),
 147                                          DMA_TO_DEVICE);
 148                         xdp_return_frame(tx_buf->xdpf);
 149                         tx_buf->action = 0;
 150                         tx_buf->xdpf = NULL;
 151                 } else if (tx_buf->action == XDP_TX) {
 152                         rx_doorbell_needed = true;
 153                         last_tx_cons = tx_cons;
 154
 155                         frags = tx_buf->nr_frags;
 156                         for (j = 0; j < frags; j++) {
 157                                 tx_cons = NEXT_TX(tx_cons);
 158                                 tx_buf = &txr->tx_buf_ring[tx_cons];
 159                                 page_pool_recycle_direct(rxr->page_pool, tx_buf->page);
 160                         }
 161                 }
 162                 tx_cons = NEXT_TX(tx_cons);
 163         }
 164         txr->tx_cons = tx_cons;
 165         if (rx_doorbell_needed) {
 166                 tx_buf = &txr->tx_buf_ring[last_tx_cons];
 167                 bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod);
 168
 169         }
 170 }
 171
 172 bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
 173 {
 174         struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
 175
 176         return !!xdp_prog;
 177 }
 178
 179 void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 180                         u16 cons, u8 *data_ptr, unsigned int len,
 181                         struct xdp_buff *xdp)
 182 {
 183         struct bnxt_sw_rx_bd *rx_buf;
 184         u32 buflen = PAGE_SIZE;
 185         struct pci_dev *pdev;
 186         dma_addr_t mapping;
 187         u32 offset;
 188
 189         pdev = bp->pdev;
 190         rx_buf = &rxr->rx_buf_ring[cons];
 191         offset = bp->rx_offset;
 192
 193         mapping = rx_buf->mapping - bp->rx_dma_offset;
 194         dma_sync_single_for_cpu(&pdev->dev, mapping + offset, len, bp->rx_dir);
 195
 196         if (bp->xdp_has_frags)
 197                 buflen = BNXT_PAGE_MODE_BUF_SIZE + offset;
 198
 199         xdp_init_buff(xdp, buflen, &rxr->xdp_rxq);
 200         xdp_prepare_buff(xdp, data_ptr - offset, offset, len, false);
 201 }
 202
 203 void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr,
 204                               struct xdp_buff *xdp)
 205 {
 206         struct skb_shared_info *shinfo;
 207         int i;
 208
 209         if (!xdp || !xdp_buff_has_frags(xdp))
 210                 return;
 211         shinfo = xdp_get_shared_info_from_buff(xdp);
 212         for (i = 0; i < shinfo->nr_frags; i++) {
 213                 struct page *page = skb_frag_page(&shinfo->frags[i]);
 214
 215                 page_pool_recycle_direct(rxr->page_pool, page);
 216         }
 217         shinfo->nr_frags = 0;
 218 }
 219
 220 /* returns the following:
 221  * true    - packet consumed by XDP and new buffer is allocated.
 222  * false   - packet should be passed to the stack.
 223  */
 224 bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
 225                  struct xdp_buff xdp, struct page *page, unsigned int *len, u8 *event)
 226 {
 227         struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
 228         struct bnxt_tx_ring_info *txr;
 229         struct bnxt_sw_rx_bd *rx_buf;
 230         struct pci_dev *pdev;
 231         dma_addr_t mapping;
 232         u32 tx_needed = 1;
 233         void *orig_data;
 234         u32 tx_avail;
 235         u32 offset;
 236         u32 act;
 237
 238         if (!xdp_prog)
 239                 return false;
 240
 241         pdev = bp->pdev;
 242         offset = bp->rx_offset;
 243
 244         txr = rxr->bnapi->tx_ring;
 245         /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */
 246         orig_data = xdp.data;
 247
 248         act = bpf_prog_run_xdp(xdp_prog, &xdp);
 249
 250         tx_avail = bnxt_tx_avail(bp, txr);
 251         /* If the tx ring is not full, we must not update the rx producer yet
 252          * because we may still be transmitting on some BDs.
 253          */
 254         if (tx_avail != bp->tx_ring_size)
 255                 *event &= ~BNXT_RX_EVENT;
 256
 257         *len = xdp.data_end - xdp.data;
 258         if (orig_data != xdp.data)
 259                 offset = xdp.data - xdp.data_hard_start;
 260
 261         switch (act) {
 262         case XDP_PASS:
 263                 return false;
 264
 265         case XDP_TX:
 266                 rx_buf = &rxr->rx_buf_ring[cons];
 267                 mapping = rx_buf->mapping - bp->rx_dma_offset;
 268                 *event = 0;
 269
 270                 if (unlikely(xdp_buff_has_frags(&xdp))) {
 271                         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(&xdp);
 272
 273                         tx_needed += sinfo->nr_frags;
 274                         *event = BNXT_AGG_EVENT;
 275                 }
 276
 277                 if (tx_avail < tx_needed) {
 278                         trace_xdp_exception(bp->dev, xdp_prog, act);
 279                         bnxt_xdp_buff_frags_free(rxr, &xdp);
 280                         bnxt_reuse_rx_data(rxr, cons, page);
 281                         return true;
 282                 }
 283
 284                 dma_sync_single_for_device(&pdev->dev, mapping + offset, *len,
 285                                            bp->rx_dir);
 286
 287                 *event |= BNXT_TX_EVENT;
 288                 __bnxt_xmit_xdp(bp, txr, mapping + offset, *len,
 289                                 NEXT_RX(rxr->rx_prod), &xdp);
 290                 bnxt_reuse_rx_data(rxr, cons, page);
 291                 return true;
 292         case XDP_REDIRECT:
 293                 /* if we are calling this here then we know that the
 294                  * redirect is coming from a frame received by the
 295                  * bnxt_en driver.
 296                  */
 297                 rx_buf = &rxr->rx_buf_ring[cons];
 298                 mapping = rx_buf->mapping - bp->rx_dma_offset;
 299                 dma_unmap_page_attrs(&pdev->dev, mapping,
 300                                      PAGE_SIZE, bp->rx_dir,
 301                                      DMA_ATTR_WEAK_ORDERING);
 302
 303                 /* if we are unable to allocate a new buffer, abort and reuse */
 304                 if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) {
 305                         trace_xdp_exception(bp->dev, xdp_prog, act);
 306                         bnxt_xdp_buff_frags_free(rxr, &xdp);
 307                         bnxt_reuse_rx_data(rxr, cons, page);
 308                         return true;
 309                 }
 310
 311                 if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) {
 312                         trace_xdp_exception(bp->dev, xdp_prog, act);
 313                         page_pool_recycle_direct(rxr->page_pool, page);
 314                         return true;
 315                 }
 316
 317                 *event |= BNXT_REDIRECT_EVENT;
 318                 break;
 319         default:
 320                 bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act);
 321                 fallthrough;
 322         case XDP_ABORTED:
 323                 trace_xdp_exception(bp->dev, xdp_prog, act);
 324                 fallthrough;
 325         case XDP_DROP:
 326                 bnxt_xdp_buff_frags_free(rxr, &xdp);
 327                 bnxt_reuse_rx_data(rxr, cons, page);
 328                 break;
 329         }
 330         return true;
 331 }
 332
 333 int bnxt_xdp_xmit(struct net_device *dev, int num_frames,
 334                   struct xdp_frame **frames, u32 flags)
 335 {
 336         struct bnxt *bp = netdev_priv(dev);
 337         struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog);
 338         struct pci_dev *pdev = bp->pdev;
 339         struct bnxt_tx_ring_info *txr;
 340         dma_addr_t mapping;
 341         int nxmit = 0;
 342         int ring;
 343         int i;
 344
 345         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
 346             !bp->tx_nr_rings_xdp ||
 347             !xdp_prog)
 348                 return -EINVAL;
 349
 350         ring = smp_processor_id() % bp->tx_nr_rings_xdp;
 351         txr = &bp->tx_ring[ring];
 352
 353         if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING)
 354                 return -EINVAL;
 355
 356         if (static_branch_unlikely(&bnxt_xdp_locking_key))
 357                 spin_lock(&txr->xdp_tx_lock);
 358
 359         for (i = 0; i < num_frames; i++) {
 360                 struct xdp_frame *xdp = frames[i];
 361
 362                 if (!bnxt_tx_avail(bp, txr))
 363                         break;
 364
 365                 mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,
 366                                          DMA_TO_DEVICE);
 367
 368                 if (dma_mapping_error(&pdev->dev, mapping))
 369                         break;
 370
 371                 __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp);
 372                 nxmit++;
 373         }
 374
 375         if (flags & XDP_XMIT_FLUSH) {
 376                 /* Sync BD data before updating doorbell */
 377                 wmb();
 378                 bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);
 379         }
 380
 381         if (static_branch_unlikely(&bnxt_xdp_locking_key))
 382                 spin_unlock(&txr->xdp_tx_lock);
 383
 384         return nxmit;
 385 }
 386
 387 /* Under rtnl_lock */
 388 static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)
 389 {
 390         struct net_device *dev = bp->dev;
 391         int tx_xdp = 0, rc, tc;
 392         struct bpf_prog *old;
 393
 394         if (prog && !prog->aux->xdp_has_frags &&
 395             bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
 396                 netdev_warn(dev, "MTU %d larger than %d without XDP frag support.\n",
 397                             bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU);
 398                 return -EOPNOTSUPP;
 399         }
 400         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) {
 401                 netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n");
 402                 return -EOPNOTSUPP;
 403         }
 404         if (prog) {
 405                 tx_xdp = bp->rx_nr_rings;
 406                 bp->xdp_has_frags = prog->aux->xdp_has_frags;
 407         }
 408
 409         tc = netdev_get_num_tc(dev);
 410         if (!tc)
 411                 tc = 1;
 412         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
 413                               true, tc, tx_xdp);
 414         if (rc) {
 415                 netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n");
 416                 return rc;
 417         }
 418         if (netif_running(dev))
 419                 bnxt_close_nic(bp, true, false);
 420
 421         old = xchg(&bp->xdp_prog, prog);
 422         if (old)
 423                 bpf_prog_put(old);
 424
 425         if (prog) {
 426                 bnxt_set_rx_skb_mode(bp, true);
 427         } else {
 428                 int rx, tx;
 429
 430                 bnxt_set_rx_skb_mode(bp, false);
 431                 bnxt_get_max_rings(bp, &rx, &tx, true);
 432                 if (rx > 1) {
 433                         bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS;
 434                         bp->dev->hw_features |= NETIF_F_LRO;
 435                 }
 436         }
 437         bp->tx_nr_rings_xdp = tx_xdp;
 438         bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp;
 439         bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
 440         bnxt_set_tpa_flags(bp);
 441         bnxt_set_ring_params(bp);
 442
 443         if (netif_running(dev))
 444                 return bnxt_open_nic(bp, true, false);
 445
 446         return 0;
 447 }
 448
 449 int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 450 {
 451         struct bnxt *bp = netdev_priv(dev);
 452         int rc;
 453
 454         switch (xdp->command) {
 455         case XDP_SETUP_PROG:
 456                 rc = bnxt_xdp_set(bp, xdp->prog);
 457                 break;
 458         default:
 459                 rc = -EINVAL;
 460                 break;
 461         }
 462         return rc;
 463 }
 464
 465 struct sk_buff *
 466 bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags,
 467                    struct page_pool *pool, struct xdp_buff *xdp,
 468                    struct rx_cmp_ext *rxcmp1)
 469 {
 470         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
 471
 472         if (!skb)
 473                 return NULL;
 474         skb_checksum_none_assert(skb);
 475         if (RX_CMP_L4_CS_OK(rxcmp1)) {
 476                 if (bp->dev->features & NETIF_F_RXCSUM) {
 477                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 478                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
 479                 }
 480         }
 481         xdp_update_skb_shared_info(skb, num_frags,
 482                                    sinfo->xdp_frags_size,
 483                                    PAGE_SIZE * sinfo->nr_frags,
 484                                    xdp_buff_is_frag_pfmemalloc(xdp));
 485         return skb;
 486 }