]>
Commit | Line | Data |
---|---|---|
1738cd3e NB |
1 | /* |
2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | ||
33 | #ifndef ENA_H | |
34 | #define ENA_H | |
35 | ||
36 | #include <linux/bitops.h> | |
282faf61 | 37 | #include <linux/dim.h> |
1738cd3e | 38 | #include <linux/etherdevice.h> |
838c93dc | 39 | #include <linux/if_vlan.h> |
1738cd3e NB |
40 | #include <linux/inetdevice.h> |
41 | #include <linux/interrupt.h> | |
42 | #include <linux/netdevice.h> | |
43 | #include <linux/skbuff.h> | |
44 | ||
45 | #include "ena_com.h" | |
46 | #include "ena_eth_com.h" | |
47 | ||
92040c6d AK |
48 | #define DRV_MODULE_GEN_MAJOR 2 |
49 | #define DRV_MODULE_GEN_MINOR 1 | |
50 | #define DRV_MODULE_GEN_SUBMINOR 0 | |
51 | ||
1738cd3e | 52 | #define DRV_MODULE_NAME "ena" |
92040c6d AK |
53 | #ifndef DRV_MODULE_GENERATION |
54 | #define DRV_MODULE_GENERATION \ | |
55 | __stringify(DRV_MODULE_GEN_MAJOR) "." \ | |
56 | __stringify(DRV_MODULE_GEN_MINOR) "." \ | |
57 | __stringify(DRV_MODULE_GEN_SUBMINOR) "K" | |
58 | #endif | |
59 | ||
1738cd3e NB |
60 | #define DEVICE_NAME "Elastic Network Adapter (ENA)" |
61 | ||
62 | /* 1 for AENQ + ADMIN */ | |
06443684 NB |
63 | #define ENA_ADMIN_MSIX_VEC 1 |
64 | #define ENA_MAX_MSIX_VEC(io_queues) (ENA_ADMIN_MSIX_VEC + (io_queues)) | |
65 | ||
bd791175 AK |
66 | /* The ENA buffer length fields is 16 bit long. So when PAGE_SIZE == 64kB the |
67 | * driver passes 0. | |
68 | * Since the max packet size the ENA handles is ~9kB limit the buffer length to | |
69 | * 16kB. | |
70 | */ | |
71 | #if PAGE_SIZE > SZ_16K | |
caec6619 | 72 | #define ENA_PAGE_SIZE (_AC(SZ_16K, UL)) |
bd791175 AK |
73 | #else |
74 | #define ENA_PAGE_SIZE PAGE_SIZE | |
75 | #endif | |
76 | ||
06443684 | 77 | #define ENA_MIN_MSIX_VEC 2 |
1738cd3e NB |
78 | |
79 | #define ENA_REG_BAR 0 | |
80 | #define ENA_MEM_BAR 2 | |
81 | #define ENA_BAR_MASK (BIT(ENA_REG_BAR) | BIT(ENA_MEM_BAR)) | |
82 | ||
83 | #define ENA_DEFAULT_RING_SIZE (1024) | |
13ca32a6 SJ |
84 | #define ENA_MIN_RING_SIZE (256) |
85 | ||
2413ea97 SJ |
86 | #define ENA_MIN_NUM_IO_QUEUES (1) |
87 | ||
1738cd3e | 88 | #define ENA_TX_WAKEUP_THRESH (MAX_SKB_FRAGS + 2) |
87731f0c | 89 | #define ENA_DEFAULT_RX_COPYBREAK (256 - NET_IP_ALIGN) |
1738cd3e NB |
90 | |
91 | /* limit the buffer size to 600 bytes to handle MTU changes from very | |
92 | * small to very large, in which case the number of buffers per packet | |
93 | * could exceed ENA_PKT_MAX_BUFS | |
94 | */ | |
95 | #define ENA_DEFAULT_MIN_RX_BUFF_ALLOC_SIZE 600 | |
96 | ||
97 | #define ENA_MIN_MTU 128 | |
98 | ||
99 | #define ENA_NAME_MAX_LEN 20 | |
100 | #define ENA_IRQNAME_SIZE 40 | |
101 | ||
102 | #define ENA_PKT_MAX_BUFS 19 | |
103 | ||
104 | #define ENA_RX_RSS_TABLE_LOG_SIZE 7 | |
105 | #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) | |
106 | ||
107 | #define ENA_HASH_KEY_SIZE 40 | |
108 | ||
109 | /* The number of tx packet completions that will be handled each NAPI poll | |
110 | * cycle is ring_size / ENA_TX_POLL_BUDGET_DIVIDER. | |
111 | */ | |
112 | #define ENA_TX_POLL_BUDGET_DIVIDER 4 | |
113 | ||
0574bb80 AK |
114 | /* Refill Rx queue when number of required descriptors is above |
115 | * QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER or ENA_RX_REFILL_THRESH_PACKET | |
1738cd3e NB |
116 | */ |
117 | #define ENA_RX_REFILL_THRESH_DIVIDER 8 | |
0574bb80 | 118 | #define ENA_RX_REFILL_THRESH_PACKET 256 |
1738cd3e NB |
119 | |
120 | /* Number of queues to check for missing queues per timer service */ | |
121 | #define ENA_MONITORED_TX_QUEUES 4 | |
122 | /* Max timeout packets before device reset */ | |
7102a18a | 123 | #define MAX_NUM_OF_TIMEOUTED_PACKETS 128 |
1738cd3e NB |
124 | |
125 | #define ENA_TX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1)) | |
126 | ||
127 | #define ENA_RX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1)) | |
128 | #define ENA_RX_RING_IDX_ADD(idx, n, ring_size) \ | |
129 | (((idx) + (n)) & ((ring_size) - 1)) | |
130 | ||
131 | #define ENA_IO_TXQ_IDX(q) (2 * (q)) | |
132 | #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) | |
92569fd2 AK |
133 | #define ENA_IO_TXQ_IDX_TO_COMBINED_IDX(q) ((q) / 2) |
134 | #define ENA_IO_RXQ_IDX_TO_COMBINED_IDX(q) (((q) - 1) / 2) | |
1738cd3e NB |
135 | |
136 | #define ENA_MGMNT_IRQ_IDX 0 | |
137 | #define ENA_IO_IRQ_FIRST_IDX 1 | |
138 | #define ENA_IO_IRQ_IDX(q) (ENA_IO_IRQ_FIRST_IDX + (q)) | |
139 | ||
140 | /* ENA device should send keep alive msg every 1 sec. | |
7102a18a | 141 | * We wait for 6 sec just to be on the safe side. |
1738cd3e | 142 | */ |
7102a18a | 143 | #define ENA_DEVICE_KALIVE_TIMEOUT (6 * HZ) |
8510e1a3 | 144 | #define ENA_MAX_NO_INTERRUPT_ITERATIONS 3 |
1738cd3e NB |
145 | |
146 | #define ENA_MMIO_DISABLE_REG_READ BIT(0) | |
147 | ||
838c93dc SJ |
148 | /* The max MTU size is configured to be the ethernet frame size without |
149 | * the overhead of the ethernet header, which can have a VLAN header, and | |
150 | * a frame check sequence (FCS). | |
151 | * The buffer size we share with the device is defined to be ENA_PAGE_SIZE | |
152 | */ | |
153 | ||
154 | #define ENA_XDP_MAX_MTU (ENA_PAGE_SIZE - ETH_HLEN - ETH_FCS_LEN - \ | |
155 | VLAN_HLEN - XDP_PACKET_HEADROOM) | |
156 | ||
548c4940 SJ |
157 | #define ENA_IS_XDP_INDEX(adapter, index) (((index) >= (adapter)->xdp_first_ring) && \ |
158 | ((index) < (adapter)->xdp_first_ring + (adapter)->xdp_num_queues)) | |
159 | ||
1738cd3e NB |
160 | struct ena_irq { |
161 | irq_handler_t handler; | |
162 | void *data; | |
163 | int cpu; | |
164 | u32 vector; | |
165 | cpumask_t affinity_hint_mask; | |
166 | char name[ENA_IRQNAME_SIZE]; | |
167 | }; | |
168 | ||
169 | struct ena_napi { | |
170 | struct napi_struct napi ____cacheline_aligned; | |
171 | struct ena_ring *tx_ring; | |
172 | struct ena_ring *rx_ring; | |
548c4940 | 173 | struct ena_ring *xdp_ring; |
913b0bfd | 174 | bool first_interrupt; |
1738cd3e | 175 | u32 qid; |
282faf61 | 176 | struct dim dim; |
1738cd3e NB |
177 | }; |
178 | ||
31aa9857 SJ |
179 | struct ena_calc_queue_size_ctx { |
180 | struct ena_com_dev_get_features_ctx *get_feat_ctx; | |
181 | struct ena_com_dev *ena_dev; | |
182 | struct pci_dev *pdev; | |
736ce3f4 SJ |
183 | u32 tx_queue_size; |
184 | u32 rx_queue_size; | |
185 | u32 max_tx_queue_size; | |
186 | u32 max_rx_queue_size; | |
31aa9857 SJ |
187 | u16 max_tx_sgl_size; |
188 | u16 max_rx_sgl_size; | |
189 | }; | |
190 | ||
1738cd3e NB |
191 | struct ena_tx_buffer { |
192 | struct sk_buff *skb; | |
193 | /* num of ena desc for this specific skb | |
194 | * (includes data desc and metadata desc) | |
195 | */ | |
196 | u32 tx_descs; | |
197 | /* num of buffers used by this skb */ | |
198 | u32 num_of_bufs; | |
800c55cb | 199 | |
548c4940 SJ |
200 | /* XDP buffer structure which is used for sending packets in |
201 | * the xdp queues | |
202 | */ | |
203 | struct xdp_frame *xdpf; | |
204 | /* The rx page for the rx buffer that was received in rx and | |
205 | * re transmitted on xdp tx queues as a result of XDP_TX action. | |
206 | * We need to free the page once we finished cleaning the buffer in | |
207 | * clean_xdp_irq() | |
208 | */ | |
209 | struct page *xdp_rx_page; | |
210 | ||
38005ca8 AK |
211 | /* Indicate if bufs[0] map the linear data of the skb. */ |
212 | u8 map_linear_data; | |
213 | ||
800c55cb NB |
214 | /* Used for detect missing tx packets to limit the number of prints */ |
215 | u32 print_once; | |
216 | /* Save the last jiffies to detect missing tx packets | |
217 | * | |
218 | * sets to non zero value on ena_start_xmit and set to zero on | |
219 | * napi and timer_Service_routine. | |
220 | * | |
221 | * while this value is not protected by lock, | |
222 | * a given packet is not expected to be handled by ena_start_xmit | |
223 | * and by napi/timer_service at the same time. | |
224 | */ | |
1738cd3e NB |
225 | unsigned long last_jiffies; |
226 | struct ena_com_buf bufs[ENA_PKT_MAX_BUFS]; | |
227 | } ____cacheline_aligned; | |
228 | ||
229 | struct ena_rx_buffer { | |
230 | struct sk_buff *skb; | |
231 | struct page *page; | |
232 | u32 page_offset; | |
233 | struct ena_com_buf ena_buf; | |
234 | } ____cacheline_aligned; | |
235 | ||
236 | struct ena_stats_tx { | |
237 | u64 cnt; | |
238 | u64 bytes; | |
239 | u64 queue_stop; | |
240 | u64 prepare_ctx_err; | |
241 | u64 queue_wakeup; | |
242 | u64 dma_mapping_err; | |
243 | u64 linearize; | |
244 | u64 linearize_failed; | |
245 | u64 napi_comp; | |
246 | u64 tx_poll; | |
247 | u64 doorbells; | |
1738cd3e | 248 | u64 bad_req_id; |
38005ca8 | 249 | u64 llq_buffer_copy; |
11095fdb | 250 | u64 missed_tx; |
1738cd3e NB |
251 | }; |
252 | ||
253 | struct ena_stats_rx { | |
254 | u64 cnt; | |
255 | u64 bytes; | |
d2eecc6e SJ |
256 | u64 rx_copybreak_pkt; |
257 | u64 csum_good; | |
1738cd3e NB |
258 | u64 refil_partial; |
259 | u64 bad_csum; | |
260 | u64 page_alloc_fail; | |
261 | u64 skb_alloc_fail; | |
262 | u64 dma_mapping_err; | |
263 | u64 bad_desc_num; | |
ad974bae | 264 | u64 bad_req_id; |
a3af7c18 | 265 | u64 empty_rx_ring; |
cb36bb36 | 266 | u64 csum_unchecked; |
1738cd3e NB |
267 | }; |
268 | ||
269 | struct ena_ring { | |
f9172498 SJ |
270 | /* Holds the empty requests for TX/RX |
271 | * out of order completions | |
272 | */ | |
273 | u16 *free_ids; | |
ad974bae | 274 | |
1738cd3e NB |
275 | union { |
276 | struct ena_tx_buffer *tx_buffer_info; | |
277 | struct ena_rx_buffer *rx_buffer_info; | |
278 | }; | |
279 | ||
280 | /* cache ptr to avoid using the adapter */ | |
281 | struct device *dev; | |
282 | struct pci_dev *pdev; | |
283 | struct napi_struct *napi; | |
284 | struct net_device *netdev; | |
285 | struct ena_com_dev *ena_dev; | |
286 | struct ena_adapter *adapter; | |
287 | struct ena_com_io_cq *ena_com_io_cq; | |
288 | struct ena_com_io_sq *ena_com_io_sq; | |
838c93dc SJ |
289 | struct bpf_prog *xdp_bpf_prog; |
290 | struct xdp_rxq_info xdp_rxq; | |
1738cd3e NB |
291 | |
292 | u16 next_to_use; | |
293 | u16 next_to_clean; | |
294 | u16 rx_copybreak; | |
838c93dc | 295 | u16 rx_headroom; |
1738cd3e NB |
296 | u16 qid; |
297 | u16 mtu; | |
298 | u16 sgl_size; | |
299 | ||
300 | /* The maximum header length the device can handle */ | |
301 | u8 tx_max_header_size; | |
302 | ||
8510e1a3 NB |
303 | bool first_interrupt; |
304 | u16 no_interrupt_event_cnt; | |
305 | ||
1738cd3e NB |
306 | /* cpu for TPH */ |
307 | int cpu; | |
308 | /* number of tx/rx_buffer_info's entries */ | |
309 | int ring_size; | |
310 | ||
311 | enum ena_admin_placement_policy_type tx_mem_queue_type; | |
312 | ||
313 | struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS]; | |
314 | u32 smoothed_interval; | |
315 | u32 per_napi_packets; | |
282faf61 | 316 | u16 non_empty_napi_events; |
1738cd3e NB |
317 | struct u64_stats_sync syncp; |
318 | union { | |
319 | struct ena_stats_tx tx_stats; | |
320 | struct ena_stats_rx rx_stats; | |
321 | }; | |
38005ca8 AK |
322 | |
323 | u8 *push_buf_intermediate_buf; | |
a3af7c18 | 324 | int empty_rx_queue; |
1738cd3e NB |
325 | } ____cacheline_aligned; |
326 | ||
327 | struct ena_stats_dev { | |
328 | u64 tx_timeout; | |
8c5c7abd NB |
329 | u64 suspend; |
330 | u64 resume; | |
1738cd3e NB |
331 | u64 wd_expired; |
332 | u64 interface_up; | |
333 | u64 interface_down; | |
334 | u64 admin_q_pause; | |
d81db240 | 335 | u64 rx_drops; |
1738cd3e NB |
336 | }; |
337 | ||
338 | enum ena_flags_t { | |
339 | ENA_FLAG_DEVICE_RUNNING, | |
340 | ENA_FLAG_DEV_UP, | |
341 | ENA_FLAG_LINK_UP, | |
06443684 | 342 | ENA_FLAG_MSIX_ENABLED, |
d18e4f68 NB |
343 | ENA_FLAG_TRIGGER_RESET, |
344 | ENA_FLAG_ONGOING_RESET | |
1738cd3e NB |
345 | }; |
346 | ||
347 | /* adapter specific private data structure */ | |
348 | struct ena_adapter { | |
349 | struct ena_com_dev *ena_dev; | |
350 | /* OS defined structs */ | |
351 | struct net_device *netdev; | |
352 | struct pci_dev *pdev; | |
353 | ||
354 | /* rx packets that shorter that this len will be copied to the skb | |
355 | * header | |
356 | */ | |
357 | u32 rx_copybreak; | |
358 | u32 max_mtu; | |
359 | ||
736ce3f4 SJ |
360 | u32 num_io_queues; |
361 | u32 max_num_io_queues; | |
1738cd3e | 362 | |
1738cd3e NB |
363 | int msix_vecs; |
364 | ||
82ef30f1 NB |
365 | u32 missing_tx_completion_threshold; |
366 | ||
13ca32a6 SJ |
367 | u32 requested_tx_ring_size; |
368 | u32 requested_rx_ring_size; | |
1738cd3e | 369 | |
31aa9857 SJ |
370 | u32 max_tx_ring_size; |
371 | u32 max_rx_ring_size; | |
372 | ||
1738cd3e NB |
373 | u32 msg_enable; |
374 | ||
375 | u16 max_tx_sgl_size; | |
376 | u16 max_rx_sgl_size; | |
377 | ||
378 | u8 mac_addr[ETH_ALEN]; | |
379 | ||
82ef30f1 NB |
380 | unsigned long keep_alive_timeout; |
381 | unsigned long missing_tx_completion_to; | |
382 | ||
1738cd3e NB |
383 | char name[ENA_NAME_MAX_LEN]; |
384 | ||
385 | unsigned long flags; | |
386 | /* TX */ | |
387 | struct ena_ring tx_ring[ENA_MAX_NUM_IO_QUEUES] | |
388 | ____cacheline_aligned_in_smp; | |
389 | ||
390 | /* RX */ | |
391 | struct ena_ring rx_ring[ENA_MAX_NUM_IO_QUEUES] | |
392 | ____cacheline_aligned_in_smp; | |
393 | ||
394 | struct ena_napi ena_napi[ENA_MAX_NUM_IO_QUEUES]; | |
395 | ||
396 | struct ena_irq irq_tbl[ENA_MAX_MSIX_VEC(ENA_MAX_NUM_IO_QUEUES)]; | |
397 | ||
398 | /* timer service */ | |
399 | struct work_struct reset_task; | |
1738cd3e NB |
400 | struct timer_list timer_service; |
401 | ||
402 | bool wd_state; | |
8c5c7abd | 403 | bool dev_up_before_reset; |
1738cd3e NB |
404 | unsigned long last_keep_alive_jiffies; |
405 | ||
406 | struct u64_stats_sync syncp; | |
407 | struct ena_stats_dev dev_stats; | |
408 | ||
409 | /* last queue index that was checked for uncompleted tx packets */ | |
410 | u32 last_monitored_tx_qid; | |
e2eed0e3 NB |
411 | |
412 | enum ena_regs_reset_reason_types reset_reason; | |
838c93dc SJ |
413 | |
414 | struct bpf_prog *xdp_bpf_prog; | |
548c4940 SJ |
415 | u32 xdp_first_ring; |
416 | u32 xdp_num_queues; | |
1738cd3e NB |
417 | }; |
418 | ||
419 | void ena_set_ethtool_ops(struct net_device *netdev); | |
420 | ||
421 | void ena_dump_stats_to_dmesg(struct ena_adapter *adapter); | |
422 | ||
423 | void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf); | |
424 | ||
eece4d2a SJ |
425 | int ena_update_queue_sizes(struct ena_adapter *adapter, |
426 | u32 new_tx_size, | |
427 | u32 new_rx_size); | |
838c93dc | 428 | |
2413ea97 | 429 | int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count); |
eece4d2a | 430 | |
1738cd3e NB |
431 | int ena_get_sset_count(struct net_device *netdev, int sset); |
432 | ||
548c4940 SJ |
433 | enum ena_xdp_errors_t { |
434 | ENA_XDP_ALLOWED = 0, | |
435 | ENA_XDP_CURRENT_MTU_TOO_LARGE, | |
436 | ENA_XDP_NO_ENOUGH_QUEUES, | |
437 | }; | |
438 | ||
439 | static inline bool ena_xdp_queues_present(struct ena_adapter *adapter) | |
440 | { | |
441 | return adapter->xdp_first_ring != 0; | |
442 | } | |
443 | ||
838c93dc SJ |
444 | static inline bool ena_xdp_present(struct ena_adapter *adapter) |
445 | { | |
446 | return !!adapter->xdp_bpf_prog; | |
447 | } | |
448 | ||
449 | static inline bool ena_xdp_present_ring(struct ena_ring *ring) | |
450 | { | |
451 | return !!ring->xdp_bpf_prog; | |
452 | } | |
453 | ||
548c4940 SJ |
454 | static inline int ena_xdp_legal_queue_count(struct ena_adapter *adapter, |
455 | u32 queues) | |
838c93dc | 456 | { |
548c4940 SJ |
457 | return 2 * queues <= adapter->max_num_io_queues; |
458 | } | |
459 | ||
460 | static inline enum ena_xdp_errors_t ena_xdp_allowed(struct ena_adapter *adapter) | |
461 | { | |
462 | enum ena_xdp_errors_t rc = ENA_XDP_ALLOWED; | |
463 | ||
464 | if (adapter->netdev->mtu > ENA_XDP_MAX_MTU) | |
465 | rc = ENA_XDP_CURRENT_MTU_TOO_LARGE; | |
466 | else if (!ena_xdp_legal_queue_count(adapter, adapter->num_io_queues)) | |
467 | rc = ENA_XDP_NO_ENOUGH_QUEUES; | |
468 | ||
469 | return rc; | |
838c93dc SJ |
470 | } |
471 | ||
1738cd3e | 472 | #endif /* !(ENA_H) */ |