]>
git.ipfire.org Git - thirdparty/linux.git/blob - drivers/net/ethernet/google/gve/gve_rx.c
1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
4 * Copyright (C) 2015-2019 Google, Inc.
8 #include "gve_adminq.h"
9 #include <linux/etherdevice.h>
11 static void gve_rx_remove_from_block(struct gve_priv
*priv
, int queue_idx
)
13 struct gve_notify_block
*block
=
14 &priv
->ntfy_blocks
[gve_rx_idx_to_ntfy(priv
, queue_idx
)];
19 static void gve_rx_free_ring(struct gve_priv
*priv
, int idx
)
21 struct gve_rx_ring
*rx
= &priv
->rx
[idx
];
22 struct device
*dev
= &priv
->pdev
->dev
;
26 gve_rx_remove_from_block(priv
, idx
);
28 bytes
= sizeof(struct gve_rx_desc
) * priv
->rx_desc_cnt
;
29 dma_free_coherent(dev
, bytes
, rx
->desc
.desc_ring
, rx
->desc
.bus
);
30 rx
->desc
.desc_ring
= NULL
;
32 dma_free_coherent(dev
, sizeof(*rx
->q_resources
),
33 rx
->q_resources
, rx
->q_resources_bus
);
34 rx
->q_resources
= NULL
;
36 gve_unassign_qpl(priv
, rx
->data
.qpl
->id
);
38 kvfree(rx
->data
.page_info
);
40 slots
= rx
->data
.mask
+ 1;
41 bytes
= sizeof(*rx
->data
.data_ring
) * slots
;
42 dma_free_coherent(dev
, bytes
, rx
->data
.data_ring
,
44 rx
->data
.data_ring
= NULL
;
45 netif_dbg(priv
, drv
, priv
->dev
, "freed rx ring %d\n", idx
);
48 static void gve_setup_rx_buffer(struct gve_rx_slot_page_info
*page_info
,
49 struct gve_rx_data_slot
*slot
,
50 dma_addr_t addr
, struct page
*page
)
52 page_info
->page
= page
;
53 page_info
->page_offset
= 0;
54 page_info
->page_address
= page_address(page
);
55 slot
->qpl_offset
= cpu_to_be64(addr
);
58 static int gve_prefill_rx_pages(struct gve_rx_ring
*rx
)
60 struct gve_priv
*priv
= rx
->gve
;
64 /* Allocate one page per Rx queue slot. Each page is split into two
65 * packet buffers, when possible we "page flip" between the two.
67 slots
= rx
->data
.mask
+ 1;
69 rx
->data
.page_info
= kvzalloc(slots
*
70 sizeof(*rx
->data
.page_info
), GFP_KERNEL
);
71 if (!rx
->data
.page_info
)
74 rx
->data
.qpl
= gve_assign_rx_qpl(priv
);
76 for (i
= 0; i
< slots
; i
++) {
77 struct page
*page
= rx
->data
.qpl
->pages
[i
];
78 dma_addr_t addr
= i
* PAGE_SIZE
;
80 gve_setup_rx_buffer(&rx
->data
.page_info
[i
],
81 &rx
->data
.data_ring
[i
], addr
, page
);
87 static void gve_rx_add_to_block(struct gve_priv
*priv
, int queue_idx
)
89 u32 ntfy_idx
= gve_rx_idx_to_ntfy(priv
, queue_idx
);
90 struct gve_notify_block
*block
= &priv
->ntfy_blocks
[ntfy_idx
];
91 struct gve_rx_ring
*rx
= &priv
->rx
[queue_idx
];
94 rx
->ntfy_id
= ntfy_idx
;
97 static int gve_rx_alloc_ring(struct gve_priv
*priv
, int idx
)
99 struct gve_rx_ring
*rx
= &priv
->rx
[idx
];
100 struct device
*hdev
= &priv
->pdev
->dev
;
106 netif_dbg(priv
, drv
, priv
->dev
, "allocating rx ring\n");
107 /* Make sure everything is zeroed to start with */
108 memset(rx
, 0, sizeof(*rx
));
113 slots
= priv
->rx_pages_per_qpl
;
114 rx
->data
.mask
= slots
- 1;
116 /* alloc rx data ring */
117 bytes
= sizeof(*rx
->data
.data_ring
) * slots
;
118 rx
->data
.data_ring
= dma_alloc_coherent(hdev
, bytes
,
121 if (!rx
->data
.data_ring
)
123 filled_pages
= gve_prefill_rx_pages(rx
);
124 if (filled_pages
< 0) {
126 goto abort_with_slots
;
128 rx
->desc
.fill_cnt
= filled_pages
;
129 /* Ensure data ring slots (packet buffers) are visible. */
132 /* Alloc gve_queue_resources */
134 dma_alloc_coherent(hdev
,
135 sizeof(*rx
->q_resources
),
136 &rx
->q_resources_bus
,
138 if (!rx
->q_resources
) {
142 netif_dbg(priv
, drv
, priv
->dev
, "rx[%d]->data.data_bus=%lx\n", idx
,
143 (unsigned long)rx
->data
.data_bus
);
145 /* alloc rx desc ring */
146 bytes
= sizeof(struct gve_rx_desc
) * priv
->rx_desc_cnt
;
147 npages
= bytes
/ PAGE_SIZE
;
148 if (npages
* PAGE_SIZE
!= bytes
) {
150 goto abort_with_q_resources
;
153 rx
->desc
.desc_ring
= dma_alloc_coherent(hdev
, bytes
, &rx
->desc
.bus
,
155 if (!rx
->desc
.desc_ring
) {
157 goto abort_with_q_resources
;
159 rx
->desc
.mask
= slots
- 1;
162 gve_rx_add_to_block(priv
, idx
);
166 abort_with_q_resources
:
167 dma_free_coherent(hdev
, sizeof(*rx
->q_resources
),
168 rx
->q_resources
, rx
->q_resources_bus
);
169 rx
->q_resources
= NULL
;
171 kvfree(rx
->data
.page_info
);
173 bytes
= sizeof(*rx
->data
.data_ring
) * slots
;
174 dma_free_coherent(hdev
, bytes
, rx
->data
.data_ring
, rx
->data
.data_bus
);
175 rx
->data
.data_ring
= NULL
;
180 int gve_rx_alloc_rings(struct gve_priv
*priv
)
185 for (i
= 0; i
< priv
->rx_cfg
.num_queues
; i
++) {
186 err
= gve_rx_alloc_ring(priv
, i
);
188 netif_err(priv
, drv
, priv
->dev
,
189 "Failed to alloc rx ring=%d: err=%d\n",
194 /* Unallocate if there was an error */
198 for (j
= 0; j
< i
; j
++)
199 gve_rx_free_ring(priv
, j
);
204 void gve_rx_free_rings(struct gve_priv
*priv
)
208 for (i
= 0; i
< priv
->rx_cfg
.num_queues
; i
++)
209 gve_rx_free_ring(priv
, i
);
212 void gve_rx_write_doorbell(struct gve_priv
*priv
, struct gve_rx_ring
*rx
)
214 u32 db_idx
= be32_to_cpu(rx
->q_resources
->db_index
);
216 iowrite32be(rx
->desc
.fill_cnt
, &priv
->db_bar2
[db_idx
]);
219 static enum pkt_hash_types
gve_rss_type(__be16 pkt_flags
)
221 if (likely(pkt_flags
& (GVE_RXF_TCP
| GVE_RXF_UDP
)))
222 return PKT_HASH_TYPE_L4
;
223 if (pkt_flags
& (GVE_RXF_IPV4
| GVE_RXF_IPV6
))
224 return PKT_HASH_TYPE_L3
;
225 return PKT_HASH_TYPE_L2
;
228 static struct sk_buff
*gve_rx_copy(struct net_device
*dev
,
229 struct napi_struct
*napi
,
230 struct gve_rx_slot_page_info
*page_info
,
233 struct sk_buff
*skb
= napi_alloc_skb(napi
, len
);
234 void *va
= page_info
->page_address
+ GVE_RX_PAD
+
235 page_info
->page_offset
;
242 skb_copy_to_linear_data(skb
, va
, len
);
244 skb
->protocol
= eth_type_trans(skb
, dev
);
248 static struct sk_buff
*gve_rx_add_frags(struct net_device
*dev
,
249 struct napi_struct
*napi
,
250 struct gve_rx_slot_page_info
*page_info
,
253 struct sk_buff
*skb
= napi_get_frags(napi
);
258 skb_add_rx_frag(skb
, 0, page_info
->page
,
259 page_info
->page_offset
+
260 GVE_RX_PAD
, len
, PAGE_SIZE
/ 2);
265 static void gve_rx_flip_buff(struct gve_rx_slot_page_info
*page_info
,
266 struct gve_rx_data_slot
*data_ring
)
268 u64 addr
= be64_to_cpu(data_ring
->qpl_offset
);
270 page_info
->page_offset
^= PAGE_SIZE
/ 2;
271 addr
^= PAGE_SIZE
/ 2;
272 data_ring
->qpl_offset
= cpu_to_be64(addr
);
275 static bool gve_rx(struct gve_rx_ring
*rx
, struct gve_rx_desc
*rx_desc
,
276 netdev_features_t feat
)
278 struct gve_rx_slot_page_info
*page_info
;
279 struct gve_priv
*priv
= rx
->gve
;
280 struct napi_struct
*napi
= &priv
->ntfy_blocks
[rx
->ntfy_id
].napi
;
281 struct net_device
*dev
= priv
->dev
;
287 /* drop this packet */
288 if (unlikely(rx_desc
->flags_seq
& GVE_RXF_ERR
))
291 len
= be16_to_cpu(rx_desc
->len
) - GVE_RX_PAD
;
292 idx
= rx
->data
.cnt
& rx
->data
.mask
;
293 page_info
= &rx
->data
.page_info
[idx
];
295 /* gvnic can only receive into registered segments. If the buffer
296 * can't be recycled, our only choice is to copy the data out of
297 * it so that we can return it to the device.
300 if (PAGE_SIZE
== 4096) {
301 if (len
<= priv
->rx_copybreak
) {
302 /* Just copy small packets */
303 skb
= gve_rx_copy(dev
, napi
, page_info
, len
);
306 if (unlikely(!gve_can_recycle_pages(dev
))) {
307 skb
= gve_rx_copy(dev
, napi
, page_info
, len
);
310 pagecount
= page_count(page_info
->page
);
311 if (pagecount
== 1) {
312 /* No part of this page is used by any SKBs; we attach
313 * the page fragment to a new SKB and pass it up the
316 skb
= gve_rx_add_frags(dev
, napi
, page_info
, len
);
319 /* Make sure the kernel stack can't release the page */
320 get_page(page_info
->page
);
321 /* "flip" to other packet buffer on this page */
322 gve_rx_flip_buff(page_info
, &rx
->data
.data_ring
[idx
]);
323 } else if (pagecount
>= 2) {
324 /* We have previously passed the other half of this
325 * page up the stack, but it has not yet been freed.
327 skb
= gve_rx_copy(dev
, napi
, page_info
, len
);
329 WARN(pagecount
< 1, "Pagecount should never be < 1");
333 skb
= gve_rx_copy(dev
, napi
, page_info
, len
);
337 /* We didn't manage to allocate an skb but we haven't had any
338 * reset worthy failures.
345 if (likely(feat
& NETIF_F_RXCSUM
)) {
346 /* NIC passes up the partial sum */
348 skb
->ip_summed
= CHECKSUM_COMPLETE
;
350 skb
->ip_summed
= CHECKSUM_NONE
;
351 skb
->csum
= csum_unfold(rx_desc
->csum
);
354 /* parse flags & pass relevant info up */
355 if (likely(feat
& NETIF_F_RXHASH
) &&
356 gve_needs_rss(rx_desc
->flags_seq
))
357 skb_set_hash(skb
, be32_to_cpu(rx_desc
->rss_hash
),
358 gve_rss_type(rx_desc
->flags_seq
));
360 if (skb_is_nonlinear(skb
))
361 napi_gro_frags(napi
);
363 napi_gro_receive(napi
, skb
);
367 static bool gve_rx_work_pending(struct gve_rx_ring
*rx
)
369 struct gve_rx_desc
*desc
;
373 next_idx
= rx
->desc
.cnt
& rx
->desc
.mask
;
374 desc
= rx
->desc
.desc_ring
+ next_idx
;
376 flags_seq
= desc
->flags_seq
;
377 /* Make sure we have synchronized the seq no with the device */
380 return (GVE_SEQNO(flags_seq
) == rx
->desc
.seqno
);
383 bool gve_clean_rx_done(struct gve_rx_ring
*rx
, int budget
,
384 netdev_features_t feat
)
386 struct gve_priv
*priv
= rx
->gve
;
387 struct gve_rx_desc
*desc
;
388 u32 cnt
= rx
->desc
.cnt
;
389 u32 idx
= cnt
& rx
->desc
.mask
;
393 desc
= rx
->desc
.desc_ring
+ idx
;
394 while ((GVE_SEQNO(desc
->flags_seq
) == rx
->desc
.seqno
) &&
395 work_done
< budget
) {
396 netif_info(priv
, rx_status
, priv
->dev
,
397 "[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
398 rx
->q_num
, idx
, desc
, desc
->flags_seq
);
399 netif_info(priv
, rx_status
, priv
->dev
,
400 "[%d] seqno=%d rx->desc.seqno=%d\n",
401 rx
->q_num
, GVE_SEQNO(desc
->flags_seq
),
403 bytes
+= be16_to_cpu(desc
->len
) - GVE_RX_PAD
;
404 if (!gve_rx(rx
, desc
, feat
))
405 gve_schedule_reset(priv
);
407 idx
= cnt
& rx
->desc
.mask
;
408 desc
= rx
->desc
.desc_ring
+ idx
;
409 rx
->desc
.seqno
= gve_next_seqno(rx
->desc
.seqno
);
416 u64_stats_update_begin(&rx
->statss
);
417 rx
->rpackets
+= work_done
;
419 u64_stats_update_end(&rx
->statss
);
421 rx
->desc
.fill_cnt
+= work_done
;
423 /* restock desc ring slots */
424 dma_wmb(); /* Ensure descs are visible before ringing doorbell */
425 gve_rx_write_doorbell(priv
, rx
);
426 return gve_rx_work_pending(rx
);
429 bool gve_rx_poll(struct gve_notify_block
*block
, int budget
)
431 struct gve_rx_ring
*rx
= block
->rx
;
432 netdev_features_t feat
;
435 feat
= block
->napi
.dev
->features
;
437 /* If budget is 0, do all the work */
442 repoll
|= gve_clean_rx_done(rx
, budget
, feat
);
444 repoll
|= gve_rx_work_pending(rx
);