#include "eea_pci.h"
#include "eea_ring.h"
+struct eea_sq_free_stats {
+ u64 packets;
+ u64 bytes;
+};
+
struct eea_tx_meta {
struct eea_tx_meta *next;
dma_addr_t dma_addr;
struct eea_tx_desc *desc;
u32 dma_len;
+ bool unmap;
+ bool unmap_single;
};
+static struct eea_tx_meta *eea_tx_meta_get(struct eea_net_tx *tx)
+{
+ struct eea_tx_meta *meta;
+
+ if (!tx->free)
+ return NULL;
+
+ meta = tx->free;
+ tx->free = meta->next;
+
+ return meta;
+}
+
+static void eea_tx_meta_put_and_unmap(struct eea_net_tx *tx,
+ struct eea_tx_meta *meta)
+{
+ struct eea_tx_meta *head;
+
+ head = meta;
+
+ while (true) {
+ if (meta->unmap) {
+ if (meta->unmap_single)
+ dma_unmap_single(tx->dma_dev, meta->dma_addr,
+ meta->dma_len, DMA_TO_DEVICE);
+ else
+ dma_unmap_page(tx->dma_dev, meta->dma_addr,
+ meta->dma_len, DMA_TO_DEVICE);
+ }
+
+ if (meta->next) {
+ meta = meta->next;
+ continue;
+ }
+
+ break;
+ }
+
+ meta->next = tx->free;
+ tx->free = head;
+}
+
+static void eea_meta_free_xmit(struct eea_net_tx *tx,
+ struct eea_tx_meta *meta,
+ int budget,
+ struct eea_tx_cdesc *desc,
+ struct eea_sq_free_stats *stats)
+{
+ struct sk_buff *skb = meta->skb;
+
+ if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && desc)) {
+ struct skb_shared_hwtstamps ts = {};
+
+ ts.hwtstamp = EEA_DESC_TS(desc) + tx->enet->hw_ts_offset;
+ skb_tstamp_tx(skb, &ts);
+ }
+
+ ++stats->packets;
+ napi_consume_skb(meta->skb, budget);
+
+ meta->data = NULL;
+}
+
+static int eea_clean_tx(struct eea_net_tx *tx, int budget)
+{
+ struct eea_sq_free_stats stats = {0};
+ struct eea_tx_cdesc *desc;
+ struct eea_tx_meta *meta;
+ int desc_n;
+ u16 id;
+
+ while (stats.packets < budget) {
+ desc = eea_ering_cq_get_desc(tx->ering);
+ if (!desc)
+ break;
+
+ id = le16_to_cpu(desc->id);
+ if (unlikely(id >= tx->ering->num)) {
+ if (net_ratelimit())
+ netdev_err(tx->enet->netdev, "tx invalid id %d\n",
+ id);
+ eea_ering_cq_ack_desc(tx->ering, 1);
+ continue;
+ }
+
+ meta = &tx->meta[id];
+
+ if (meta->data) {
+ eea_tx_meta_put_and_unmap(tx, meta);
+ eea_meta_free_xmit(tx, meta, budget, desc, &stats);
+ desc_n = meta->num;
+ } else {
+ if (net_ratelimit())
+ netdev_err(tx->enet->netdev,
+ "tx meta->data is null. id %d num: %d\n",
+ meta->id, meta->num);
+ desc_n = 1;
+ }
+
+ eea_ering_cq_ack_desc(tx->ering, desc_n);
+ }
+
+ return stats.packets;
+}
+
int eea_poll_tx(struct eea_net_tx *tx, int budget)
{
- /* Empty function; will be implemented in a subsequent commit. */
- return budget;
+ struct eea_net *enet = tx->enet;
+ u32 index = tx - enet->tx;
+ struct netdev_queue *txq;
+ int num;
+
+ txq = netdev_get_tx_queue(enet->netdev, index);
+
+ __netif_tx_lock(txq, smp_processor_id());
+
+ num = eea_clean_tx(tx, budget);
+
+ if (netif_tx_queue_stopped(txq) &&
+ tx->ering->num_free >= MAX_SKB_FRAGS + 2)
+ netif_tx_wake_queue(txq);
+
+ __netif_tx_unlock(txq);
+
+ return num;
+}
+
+static int eea_fill_desc_from_skb(const struct sk_buff *skb,
+ struct eea_tx_desc *desc)
+{
+ if (skb_is_gso(skb)) {
+ struct skb_shared_info *sinfo = skb_shinfo(skb);
+
+ desc->gso_size = cpu_to_le16(sinfo->gso_size);
+ if (sinfo->gso_type & SKB_GSO_TCPV4)
+ desc->gso_type = EEA_TX_GSO_TCPV4;
+
+ else if (sinfo->gso_type & SKB_GSO_TCPV6)
+ desc->gso_type = EEA_TX_GSO_TCPV6;
+
+ else if (sinfo->gso_type & SKB_GSO_UDP_L4)
+ desc->gso_type = EEA_TX_GSO_UDP_L4;
+
+ else
+ return -EINVAL;
+
+ if (sinfo->gso_type & SKB_GSO_TCP_ECN)
+ desc->gso_type |= EEA_TX_GSO_ECN;
+ } else {
+ desc->gso_type = EEA_TX_GSO_NONE;
+ }
+
+ if (skb->ip_summed == CHECKSUM_PARTIAL) {
+ desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
+ desc->csum_offset = cpu_to_le16(skb->csum_offset);
+ }
+
+ return 0;
+}
+
+static struct eea_tx_meta *__eea_tx_desc_fill(struct eea_net_tx *tx,
+ struct eea_tx_meta *head_meta,
+ dma_addr_t addr, u32 data_len,
+ u32 dma_len, bool last,
+ void *data, u16 flags,
+ bool unmap)
+{
+ struct eea_tx_meta *meta;
+ struct eea_tx_desc *desc;
+
+ meta = eea_tx_meta_get(tx);
+
+ desc = eea_ering_sq_alloc_desc(tx->ering, meta->id, last, flags);
+ desc->addr = cpu_to_le64(addr);
+ desc->len = cpu_to_le16(data_len);
+
+ meta->next = NULL;
+ meta->dma_len = dma_len;
+ meta->dma_addr = addr;
+ meta->data = data;
+ meta->num = 1;
+ meta->desc = desc;
+ meta->unmap = unmap;
+ meta->unmap_single = false;
+
+ if (head_meta) {
+ meta->next = head_meta->next;
+ head_meta->next = meta;
+ ++head_meta->num;
+ }
+
+ return meta;
+}
+
+static struct eea_tx_meta *eea_tx_desc_fill(struct eea_net_tx *tx,
+ struct eea_tx_meta *head_meta,
+ dma_addr_t addr, u32 length,
+ bool is_last, void *data, u16 flags)
+{
+ struct eea_tx_meta *meta;
+ u16 len, last;
+
+ WARN_ON_ONCE(length >= 2 * USHRT_MAX);
+
+ /* Since eea does not support BIG TCP, the maximum GSO size is capped at
+ * 64KB. Consequently, a single skb buffer (head or fragment) will not
+ * require more than two descriptors
+ */
+ if (length > USHRT_MAX) {
+ len = USHRT_MAX;
+ last = false;
+ } else {
+ len = length;
+ last = is_last;
+ }
+
+ meta = __eea_tx_desc_fill(tx, head_meta, addr, len, length,
+ last, data, flags, true);
+
+ if (length > USHRT_MAX) {
+ if (!head_meta)
+ head_meta = meta;
+
+ addr += USHRT_MAX;
+ len = length - USHRT_MAX;
+
+ __eea_tx_desc_fill(tx, head_meta, addr, len, 0, is_last,
+ NULL, 0, false);
+ }
+
+ return meta;
+}
+
+static int eea_tx_add_skb_frag(struct eea_net_tx *tx,
+ struct eea_tx_meta *head_meta,
+ const skb_frag_t *frag, bool is_last)
+{
+ u32 len = skb_frag_size(frag);
+ dma_addr_t addr;
+
+ addr = skb_frag_dma_map(tx->dma_dev, frag, 0, len, DMA_TO_DEVICE);
+ if (unlikely(dma_mapping_error(tx->dma_dev, addr)))
+ return -ENOMEM;
+
+ eea_tx_desc_fill(tx, head_meta, addr, len, is_last, NULL, 0);
+
+ return 0;
+}
+
+static int eea_tx_post_skb(struct eea_net_tx *tx, struct sk_buff *skb)
+{
+ const struct skb_shared_info *shinfo = skb_shinfo(skb);
+ u32 hlen = skb_headlen(skb);
+ struct eea_tx_meta *meta;
+ const skb_frag_t *frag;
+ dma_addr_t addr;
+ u32 len = hlen;
+ int i, err;
+ u16 flags;
+ bool last;
+
+ if (len) {
+ addr = dma_map_single(tx->dma_dev, skb->data, len,
+ DMA_TO_DEVICE);
+ if (unlikely(dma_mapping_error(tx->dma_dev, addr)))
+ return -ENOMEM;
+
+ last = !shinfo->nr_frags;
+ i = 0;
+ } else {
+ /* The net stack will never submit an skb with an skb->len of
+ * 0. If the head len is 0, the number of frags must be greater
+ * than 0.
+ */
+ frag = &shinfo->frags[0];
+ len = skb_frag_size(frag);
+
+ addr = skb_frag_dma_map(tx->dma_dev, frag, 0, len,
+ DMA_TO_DEVICE);
+ if (unlikely(dma_mapping_error(tx->dma_dev, addr)))
+ return -ENOMEM;
+
+ last = shinfo->nr_frags == 1;
+ i = 1;
+ }
+
+ flags = skb->ip_summed == CHECKSUM_PARTIAL ? EEA_DESC_F_DO_CSUM : 0;
+
+ meta = eea_tx_desc_fill(tx, NULL, addr, len, last, skb, flags);
+ meta->unmap_single = !!hlen;
+
+ err = eea_fill_desc_from_skb(skb, meta->desc);
+ if (err)
+ goto err_cancel;
+
+ for (; i < shinfo->nr_frags; i++) {
+ frag = &shinfo->frags[i];
+ bool is_last = i == (shinfo->nr_frags - 1);
+
+ err = eea_tx_add_skb_frag(tx, meta, frag, is_last);
+ if (err)
+ goto err_cancel;
+ }
+
+ eea_ering_sq_commit_desc(tx->ering);
+
+ return 0;
+
+err_cancel:
+ eea_ering_sq_cancel(tx->ering);
+ eea_tx_meta_put_and_unmap(tx, meta);
+ meta->data = NULL;
+ return err;
+}
+
+static void eea_tx_kick(struct eea_net_tx *tx)
+{
+ eea_ering_kick(tx->ering);
+}
+
+static int eea_tx_check_free_num(struct eea_net_tx *tx,
+ struct netdev_queue *txq)
+{
+ int n;
+
+ /* MAX_SKB_FRAGS + 1: Covers the skb linear head and all paged fragments
+ * 1: Extra slot for a head or fragment that exceeds 64KB.
+ */
+ n = MAX_SKB_FRAGS + 2;
+ return netif_txq_maybe_stop(txq, tx->ering->num_free, n, n);
}
netdev_tx_t eea_tx_xmit(struct sk_buff *skb, struct net_device *netdev)
{
- /* Empty function; will be implemented in a subsequent commit. */
- dev_kfree_skb_any(skb);
+ struct eea_net *enet = netdev_priv(netdev);
+ int qnum = skb_get_queue_mapping(skb);
+ struct eea_net_tx *tx = &enet->tx[qnum];
+ struct netdev_queue *txq;
+ int err, enable;
+
+ txq = netdev_get_tx_queue(netdev, qnum);
+
+ enable = eea_tx_check_free_num(tx, txq);
+ if (!enable)
+ return NETDEV_TX_BUSY;
+
+ err = eea_tx_post_skb(tx, skb);
+ if (unlikely(err)) {
+ dev_kfree_skb_any(skb);
+ } else {
+ if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
+ skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+ skb_tx_timestamp(skb);
+ }
+
+ /* NETDEV_TX_BUSY is expensive. So stop advancing the TX queue. */
+ eea_tx_check_free_num(tx, txq);
+
+ if (!netdev_xmit_more() || netif_xmit_stopped(txq))
+ eea_tx_kick(tx);
+
return NETDEV_TX_OK;
}
static void eea_free_meta(struct eea_net_tx *tx, struct eea_net_cfg *cfg)
{
+ struct eea_sq_free_stats stats = {0};
+ struct eea_tx_meta *meta;
+ int i;
+
+ while ((meta = eea_tx_meta_get(tx)))
+ meta->skb = NULL;
+
+ for (i = 0; i < cfg->tx_ring_depth; i++) {
+ meta = &tx->meta[i];
+
+ if (!meta->skb)
+ continue;
+
+ eea_tx_meta_put_and_unmap(tx, meta);
+
+ eea_meta_free_xmit(tx, meta, 0, NULL, &stats);
+ }
+
kvfree(tx->meta);
tx->meta = NULL;
}
projects / thirdparty / kernel / linux.git / commitdiff
