#include <linux/if.h>
#include <net/ip.h>
#include <net/tcp.h>
+#include <net/gro.h>
#include <linux/skbuff.h>
#include <net/page_pool/helpers.h>
#include <linux/if_vlan.h>
return IRQ_HANDLED;
}
+static struct rx_agg_cmp *bnge_get_tpa_agg(struct bnge_net *bn,
+ struct bnge_rx_ring_info *rxr,
+ u16 agg_id, u16 curr)
+{
+ struct bnge_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
+
+ return &tpa_info->agg_arr[curr];
+}
+
static struct rx_agg_cmp *bnge_get_agg(struct bnge_net *bn,
struct bnge_cp_ring_info *cpr,
u16 cp_cons, u16 curr)
}
static void bnge_reuse_rx_agg_bufs(struct bnge_cp_ring_info *cpr, u16 idx,
- u16 start, u32 agg_bufs)
+ u16 start, u32 agg_bufs, bool tpa)
{
struct bnge_napi *bnapi = cpr->bnapi;
struct bnge_net *bn = bnapi->bn;
netmem_ref netmem;
u16 cons;
- agg = bnge_get_agg(bn, cpr, idx, start + i);
+ if (tpa)
+ agg = bnge_get_tpa_agg(bn, rxr, idx, start + i);
+ else
+ agg = bnge_get_agg(bn, cpr, idx, start + i);
cons = agg->rx_agg_cmp_opaque;
__clear_bit(cons, rxr->rx_agg_bmap);
agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
RX_CMP_AGG_BUFS) >>
RX_CMP_AGG_BUFS_SHIFT;
+ } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
+ return 0;
}
if (agg_bufs) {
static u32 __bnge_rx_agg_netmems(struct bnge_net *bn,
struct bnge_cp_ring_info *cpr,
- u16 idx, u32 agg_bufs,
+ u16 idx, u32 agg_bufs, bool tpa,
struct sk_buff *skb)
{
struct bnge_napi *bnapi = cpr->bnapi;
u16 cons, frag_len;
netmem_ref netmem;
- agg = bnge_get_agg(bn, cpr, idx, i);
+ if (tpa)
+ agg = bnge_get_tpa_agg(bn, rxr, idx, i);
+ else
+ agg = bnge_get_agg(bn, cpr, idx, i);
cons = agg->rx_agg_cmp_opaque;
frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
* allocated already.
*/
rxr->rx_agg_prod = prod;
- bnge_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i);
+ bnge_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
return 0;
}
static struct sk_buff *bnge_rx_agg_netmems_skb(struct bnge_net *bn,
struct bnge_cp_ring_info *cpr,
struct sk_buff *skb, u16 idx,
- u32 agg_bufs)
+ u32 agg_bufs, bool tpa)
{
u32 total_frag_len;
- total_frag_len = __bnge_rx_agg_netmems(bn, cpr, idx, agg_bufs, skb);
+ total_frag_len = __bnge_rx_agg_netmems(bn, cpr, idx, agg_bufs,
+ tpa, skb);
if (!total_frag_len) {
skb_mark_for_recycle(skb);
dev_kfree_skb(skb);
/* TODO: Initiate reset task */
}
+static u16 bnge_tpa_alloc_agg_idx(struct bnge_rx_ring_info *rxr, u16 agg_id)
+{
+ struct bnge_tpa_idx_map *map = rxr->rx_tpa_idx_map;
+ u16 idx = agg_id & MAX_TPA_MASK;
+
+ if (test_bit(idx, map->agg_idx_bmap)) {
+ idx = find_first_zero_bit(map->agg_idx_bmap, MAX_TPA);
+ if (idx >= MAX_TPA)
+ return INVALID_HW_RING_ID;
+ }
+ __set_bit(idx, map->agg_idx_bmap);
+ map->agg_id_tbl[agg_id] = idx;
+ return idx;
+}
+
+static void bnge_free_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx)
+{
+ struct bnge_tpa_idx_map *map = rxr->rx_tpa_idx_map;
+
+ __clear_bit(idx, map->agg_idx_bmap);
+}
+
+static u16 bnge_lookup_agg_idx(struct bnge_rx_ring_info *rxr, u16 agg_id)
+{
+ struct bnge_tpa_idx_map *map = rxr->rx_tpa_idx_map;
+
+ return map->agg_id_tbl[agg_id];
+}
+
+static void bnge_tpa_metadata(struct bnge_tpa_info *tpa_info,
+ struct rx_tpa_start_cmp *tpa_start,
+ struct rx_tpa_start_cmp_ext *tpa_start1)
+{
+ tpa_info->cfa_code_valid = 1;
+ tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
+ tpa_info->vlan_valid = 0;
+ if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
+ tpa_info->vlan_valid = 1;
+ tpa_info->metadata =
+ le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
+ }
+}
+
+static void bnge_tpa_metadata_v2(struct bnge_tpa_info *tpa_info,
+ struct rx_tpa_start_cmp *tpa_start,
+ struct rx_tpa_start_cmp_ext *tpa_start1)
+{
+ tpa_info->vlan_valid = 0;
+ if (TPA_START_VLAN_VALID(tpa_start)) {
+ u32 tpid_sel = TPA_START_VLAN_TPID_SEL(tpa_start);
+ u32 vlan_proto = ETH_P_8021Q;
+
+ tpa_info->vlan_valid = 1;
+ if (tpid_sel == RX_TPA_START_METADATA1_TPID_8021AD)
+ vlan_proto = ETH_P_8021AD;
+ tpa_info->metadata = vlan_proto << 16 |
+ TPA_START_METADATA0_TCI(tpa_start1);
+ }
+}
+
+static void bnge_tpa_start(struct bnge_net *bn, struct bnge_rx_ring_info *rxr,
+ u8 cmp_type, struct rx_tpa_start_cmp *tpa_start,
+ struct rx_tpa_start_cmp_ext *tpa_start1)
+{
+ struct bnge_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
+ struct bnge_tpa_info *tpa_info;
+ u16 cons, prod, agg_id;
+ struct rx_bd *prod_bd;
+ dma_addr_t mapping;
+
+ agg_id = TPA_START_AGG_ID(tpa_start);
+ agg_id = bnge_tpa_alloc_agg_idx(rxr, agg_id);
+ if (unlikely(agg_id == INVALID_HW_RING_ID)) {
+ netdev_warn(bn->netdev, "Unable to allocate agg ID for ring %d, agg 0x%lx\n",
+ rxr->bnapi->index, TPA_START_AGG_ID(tpa_start));
+ bnge_sched_reset_rxr(bn, rxr);
+ return;
+ }
+ cons = tpa_start->rx_tpa_start_cmp_opaque;
+ prod = rxr->rx_prod;
+ cons_rx_buf = &rxr->rx_buf_ring[cons];
+ prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)];
+ tpa_info = &rxr->rx_tpa[agg_id];
+
+ if (unlikely(cons != rxr->rx_next_cons ||
+ TPA_START_ERROR(tpa_start))) {
+ netdev_warn(bn->netdev, "TPA cons %x, expected cons %x, error code %lx\n",
+ cons, rxr->rx_next_cons,
+ TPA_START_ERROR_CODE(tpa_start1));
+ bnge_sched_reset_rxr(bn, rxr);
+ return;
+ }
+ prod_rx_buf->data = tpa_info->data;
+ prod_rx_buf->data_ptr = tpa_info->data_ptr;
+
+ mapping = tpa_info->mapping;
+ prod_rx_buf->mapping = mapping;
+
+ prod_bd = &rxr->rx_desc_ring[RX_RING(bn, prod)][RX_IDX(prod)];
+
+ prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
+
+ tpa_info->data = cons_rx_buf->data;
+ tpa_info->data_ptr = cons_rx_buf->data_ptr;
+ cons_rx_buf->data = NULL;
+ tpa_info->mapping = cons_rx_buf->mapping;
+
+ tpa_info->len =
+ le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
+ RX_TPA_START_CMP_LEN_SHIFT;
+ if (likely(TPA_START_HASH_VALID(tpa_start))) {
+ tpa_info->hash_type = PKT_HASH_TYPE_L4;
+ if (TPA_START_IS_IPV6(tpa_start1))
+ tpa_info->gso_type = SKB_GSO_TCPV6;
+ else
+ tpa_info->gso_type = SKB_GSO_TCPV4;
+ tpa_info->rss_hash =
+ le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
+ } else {
+ tpa_info->hash_type = PKT_HASH_TYPE_NONE;
+ tpa_info->gso_type = 0;
+ netif_warn(bn, rx_err, bn->netdev, "TPA packet without valid hash\n");
+ }
+ tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
+ tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
+ if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP)
+ bnge_tpa_metadata(tpa_info, tpa_start, tpa_start1);
+ else
+ bnge_tpa_metadata_v2(tpa_info, tpa_start, tpa_start1);
+ tpa_info->agg_count = 0;
+
+ rxr->rx_prod = NEXT_RX(prod);
+ cons = RING_RX(bn, NEXT_RX(cons));
+ rxr->rx_next_cons = RING_RX(bn, NEXT_RX(cons));
+ cons_rx_buf = &rxr->rx_buf_ring[cons];
+
+ bnge_reuse_rx_data(rxr, cons, cons_rx_buf->data);
+ rxr->rx_prod = NEXT_RX(rxr->rx_prod);
+ cons_rx_buf->data = NULL;
+}
+
+static void bnge_abort_tpa(struct bnge_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
+{
+ if (agg_bufs)
+ bnge_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
+}
+
+static void bnge_tpa_agg(struct bnge_net *bn, struct bnge_rx_ring_info *rxr,
+ struct rx_agg_cmp *rx_agg)
+{
+ u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
+ struct bnge_tpa_info *tpa_info;
+
+ agg_id = bnge_lookup_agg_idx(rxr, agg_id);
+ tpa_info = &rxr->rx_tpa[agg_id];
+
+ if (unlikely(tpa_info->agg_count >= MAX_SKB_FRAGS)) {
+ netdev_warn(bn->netdev,
+ "TPA completion count %d exceeds limit for ring %d\n",
+ tpa_info->agg_count, rxr->bnapi->index);
+
+ bnge_sched_reset_rxr(bn, rxr);
+ return;
+ }
+
+ tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
+}
+
void bnge_reuse_rx_data(struct bnge_rx_ring_info *rxr, u16 cons, void *data)
{
struct bnge_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
return skb;
}
+#ifdef CONFIG_INET
+static void bnge_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
+{
+ struct udphdr *uh = NULL;
+
+ if (ip_proto == htons(ETH_P_IP)) {
+ struct iphdr *iph = (struct iphdr *)skb->data;
+
+ if (iph->protocol == IPPROTO_UDP)
+ uh = (struct udphdr *)(iph + 1);
+ } else {
+ struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
+
+ if (iph->nexthdr == IPPROTO_UDP)
+ uh = (struct udphdr *)(iph + 1);
+ }
+ if (uh) {
+ if (uh->check)
+ skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
+ else
+ skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
+ }
+}
+
+static struct sk_buff *bnge_gro_func(struct bnge_tpa_info *tpa_info,
+ int payload_off, int tcp_ts,
+ struct sk_buff *skb)
+{
+ u16 outer_ip_off, inner_ip_off, inner_mac_off;
+ u32 hdr_info = tpa_info->hdr_info;
+ int iphdr_len, nw_off;
+
+ inner_ip_off = BNGE_TPA_INNER_L3_OFF(hdr_info);
+ inner_mac_off = BNGE_TPA_INNER_L2_OFF(hdr_info);
+ outer_ip_off = BNGE_TPA_OUTER_L3_OFF(hdr_info);
+
+ nw_off = inner_ip_off - ETH_HLEN;
+ skb_set_network_header(skb, nw_off);
+ iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
+ sizeof(struct ipv6hdr) : sizeof(struct iphdr);
+ skb_set_transport_header(skb, nw_off + iphdr_len);
+
+ if (inner_mac_off) { /* tunnel */
+ __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
+ ETH_HLEN - 2));
+
+ bnge_gro_tunnel(skb, proto);
+ }
+
+ return skb;
+}
+
+static struct sk_buff *bnge_gro_skb(struct bnge_net *bn,
+ struct bnge_tpa_info *tpa_info,
+ struct rx_tpa_end_cmp *tpa_end,
+ struct rx_tpa_end_cmp_ext *tpa_end1,
+ struct sk_buff *skb)
+{
+ int payload_off;
+ u16 segs;
+
+ segs = TPA_END_TPA_SEGS(tpa_end);
+ if (segs == 1)
+ return skb;
+
+ NAPI_GRO_CB(skb)->count = segs;
+ skb_shinfo(skb)->gso_size =
+ le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
+ skb_shinfo(skb)->gso_type = tpa_info->gso_type;
+ payload_off = TPA_END_PAYLOAD_OFF(tpa_end1);
+ skb = bnge_gro_func(tpa_info, payload_off,
+ TPA_END_GRO_TS(tpa_end), skb);
+ if (likely(skb))
+ tcp_gro_complete(skb);
+
+ return skb;
+}
+#endif
+
+static struct sk_buff *bnge_tpa_end(struct bnge_net *bn,
+ struct bnge_cp_ring_info *cpr,
+ u32 *raw_cons,
+ struct rx_tpa_end_cmp *tpa_end,
+ struct rx_tpa_end_cmp_ext *tpa_end1,
+ u8 *event)
+{
+ struct bnge_napi *bnapi = cpr->bnapi;
+ struct net_device *dev = bn->netdev;
+ struct bnge_tpa_info *tpa_info;
+ struct bnge_rx_ring_info *rxr;
+ u8 *data_ptr, agg_bufs;
+ struct sk_buff *skb;
+ u16 idx = 0, agg_id;
+ dma_addr_t mapping;
+ unsigned int len;
+ void *data;
+
+ if (unlikely(bnapi->in_reset)) {
+ int rc = bnge_discard_rx(bn, cpr, raw_cons, tpa_end);
+
+ if (rc < 0)
+ return ERR_PTR(-EBUSY);
+ return NULL;
+ }
+
+ rxr = bnapi->rx_ring;
+ agg_id = TPA_END_AGG_ID(tpa_end);
+ agg_id = bnge_lookup_agg_idx(rxr, agg_id);
+ agg_bufs = TPA_END_AGG_BUFS(tpa_end1);
+ tpa_info = &rxr->rx_tpa[agg_id];
+ if (unlikely(agg_bufs != tpa_info->agg_count)) {
+ netdev_warn(bn->netdev, "TPA end agg_buf %d != expected agg_bufs %d\n",
+ agg_bufs, tpa_info->agg_count);
+ agg_bufs = tpa_info->agg_count;
+ }
+ tpa_info->agg_count = 0;
+ *event |= BNGE_AGG_EVENT;
+ bnge_free_agg_idx(rxr, agg_id);
+ idx = agg_id;
+ data = tpa_info->data;
+ data_ptr = tpa_info->data_ptr;
+ prefetch(data_ptr);
+ len = tpa_info->len;
+ mapping = tpa_info->mapping;
+
+ if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
+ bnge_abort_tpa(cpr, idx, agg_bufs);
+ if (agg_bufs > MAX_SKB_FRAGS)
+ netdev_warn(bn->netdev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
+ agg_bufs, (int)MAX_SKB_FRAGS);
+ return NULL;
+ }
+
+ if (len <= bn->rx_copybreak) {
+ skb = bnge_copy_skb(bnapi, data_ptr, len, mapping);
+ if (!skb) {
+ bnge_abort_tpa(cpr, idx, agg_bufs);
+ return NULL;
+ }
+ } else {
+ dma_addr_t new_mapping;
+ u8 *new_data;
+
+ new_data = __bnge_alloc_rx_frag(bn, &new_mapping, rxr,
+ GFP_ATOMIC);
+ if (!new_data) {
+ bnge_abort_tpa(cpr, idx, agg_bufs);
+ return NULL;
+ }
+
+ tpa_info->data = new_data;
+ tpa_info->data_ptr = new_data + bn->rx_offset;
+ tpa_info->mapping = new_mapping;
+
+ skb = napi_build_skb(data, bn->rx_buf_size);
+ dma_sync_single_for_cpu(bn->bd->dev, mapping,
+ bn->rx_buf_use_size, bn->rx_dir);
+
+ if (!skb) {
+ page_pool_free_va(rxr->head_pool, data, true);
+ bnge_abort_tpa(cpr, idx, agg_bufs);
+ return NULL;
+ }
+ skb_mark_for_recycle(skb);
+ skb_reserve(skb, bn->rx_offset);
+ skb_put(skb, len);
+ }
+
+ if (agg_bufs) {
+ skb = bnge_rx_agg_netmems_skb(bn, cpr, skb, idx, agg_bufs,
+ true);
+ /* Page reuse already handled by bnge_rx_agg_netmems_skb(). */
+ if (!skb)
+ return NULL;
+ }
+
+ skb->protocol = eth_type_trans(skb, dev);
+
+ if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
+ skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
+
+ if (tpa_info->vlan_valid &&
+ (dev->features & BNGE_HW_FEATURE_VLAN_ALL_RX)) {
+ __be16 vlan_proto = htons(tpa_info->metadata >>
+ RX_CMP_FLAGS2_METADATA_TPID_SFT);
+ u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
+
+ if (eth_type_vlan(vlan_proto)) {
+ __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
+ } else {
+ dev_kfree_skb(skb);
+ return NULL;
+ }
+ }
+
+ skb_checksum_none_assert(skb);
+ if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ skb->csum_level =
+ (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
+ }
+
+#ifdef CONFIG_INET
+ if (bn->priv_flags & BNGE_NET_EN_GRO)
+ skb = bnge_gro_skb(bn, tpa_info, tpa_end, tpa_end1, skb);
+#endif
+
+ return skb;
+}
+
static enum pkt_hash_types bnge_rss_ext_op(struct bnge_net *bn,
struct rx_cmp *rxcmp)
{
/* returns the following:
* 1 - 1 packet successfully received
+ * 0 - successful TPA_START, packet not completed yet
* -EBUSY - completion ring does not have all the agg buffers yet
* -ENOMEM - packet aborted due to out of memory
* -EIO - packet aborted due to hw error indicated in BD
cmp_type = RX_CMP_TYPE(rxcmp);
+ if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
+ bnge_tpa_agg(bn, rxr, (struct rx_agg_cmp *)rxcmp);
+ goto next_rx_no_prod_no_len;
+ }
+
tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
cp_cons = RING_CMP(bn, tmp_raw_cons);
rxcmp1 = (struct rx_cmp_ext *)
dma_rmb();
prod = rxr->rx_prod;
+ if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP ||
+ cmp_type == CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
+ bnge_tpa_start(bn, rxr, cmp_type,
+ (struct rx_tpa_start_cmp *)rxcmp,
+ (struct rx_tpa_start_cmp_ext *)rxcmp1);
+
+ *event |= BNGE_RX_EVENT;
+ goto next_rx_no_prod_no_len;
+
+ } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
+ skb = bnge_tpa_end(bn, cpr, &tmp_raw_cons,
+ (struct rx_tpa_end_cmp *)rxcmp,
+ (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
+ if (IS_ERR(skb))
+ return -EBUSY;
+
+ rc = -ENOMEM;
+ if (likely(skb)) {
+ bnge_deliver_skb(bn, bnapi, skb);
+ rc = 1;
+ }
+ *event |= BNGE_RX_EVENT;
+ goto next_rx_no_prod_no_len;
+ }
+
cons = rxcmp->rx_cmp_opaque;
if (unlikely(cons != rxr->rx_next_cons)) {
int rc1 = bnge_discard_rx(bn, cpr, &tmp_raw_cons, rxcmp);
if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
bnge_reuse_rx_data(rxr, cons, data);
if (agg_bufs)
- bnge_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs);
+ bnge_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
+ false);
rc = -EIO;
goto next_rx_no_len;
}
if (!skb) {
if (agg_bufs)
- bnge_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs);
+ bnge_reuse_rx_agg_bufs(cpr, cp_cons, 0,
+ agg_bufs, false);
goto oom_next_rx;
}
if (agg_bufs) {
skb = bnge_rx_agg_netmems_skb(bn, cpr, skb, cp_cons,
- agg_bufs);
+ agg_bufs, false);
if (!skb)
goto oom_next_rx;
}
cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
rxcmp1->rx_cmp_cfa_code_errors_v2 |=
cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
+ } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
+ struct rx_tpa_end_cmp_ext *tpa_end1;
+
+ tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
+ tpa_end1->rx_tpa_end_cmp_errors_v2 |=
+ cpu_to_le32(RX_TPA_END_CMP_ERRORS);
}
rc = bnge_rx_pkt(bn, cpr, raw_cons, event);
return rc;