]> git.ipfire.org Git - thirdparty/kernel/stable.git/blame - drivers/net/ethernet/mellanox/mlx5/core/en_main.c
projects / thirdparty / kernel / stable.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcp: implement TSQ for retransmits
[thirdparty/kernel/stable.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_main.c
CommitLineData
f62b8bb8 1/*
b3f63c3d 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
f62b8bb8
AV		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
e8f887ac
AV		 33#include <net/tc_act/tc_gact.h>
34#include <net/pkt_cls.h>
86d722ad 35#include <linux/mlx5/fs.h>
b3f63c3d 36#include <net/vxlan.h>
f62b8bb8 37#include "en.h"
e8f887ac 38#include "en_tc.h"
66e49ded 39#include "eswitch.h"
b3f63c3d 40#include "vxlan.h"
f62b8bb8
AV		 41
42struct mlx5e_rq_param {
cb3c7fd4
GR		 43 u32 rqc[MLX5_ST_SZ_DW(rqc)];
44 struct mlx5_wq_param wq;
45 bool am_enabled;
f62b8bb8
AV		 46};
47
48struct mlx5e_sq_param {
49 u32 sqc[MLX5_ST_SZ_DW(sqc)];
50 struct mlx5_wq_param wq;
58d52291 51 u16 max_inline;
cff92d7c 52 u8 min_inline_mode;
d3c9bc27 53 bool icosq;
f62b8bb8
AV		 54};
55
56struct mlx5e_cq_param {
57 u32 cqc[MLX5_ST_SZ_DW(cqc)];
58 struct mlx5_wq_param wq;
59 u16 eq_ix;
9908aa29 60 u8 cq_period_mode;
f62b8bb8
AV		 61};
62
63struct mlx5e_channel_param {
64 struct mlx5e_rq_param rq;
65 struct mlx5e_sq_param sq;
d3c9bc27 66 struct mlx5e_sq_param icosq;
f62b8bb8
AV		 67 struct mlx5e_cq_param rx_cq;
68 struct mlx5e_cq_param tx_cq;
d3c9bc27 69 struct mlx5e_cq_param icosq_cq;
f62b8bb8
AV		 70};
71
72static void mlx5e_update_carrier(struct mlx5e_priv *priv)
73{
74 struct mlx5_core_dev *mdev = priv->mdev;
75 u8 port_state;
76
77 port_state = mlx5_query_vport_state(mdev,
e7546514 78 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
f62b8bb8 79
87424ad5
SD		 80 if (port_state == VPORT_STATE_UP) {
81 netdev_info(priv->netdev, "Link up\n");
f62b8bb8 82 netif_carrier_on(priv->netdev);
87424ad5
SD		 83 } else {
84 netdev_info(priv->netdev, "Link down\n");
f62b8bb8 85 netif_carrier_off(priv->netdev);
87424ad5 86 }
f62b8bb8
AV		 87}
88
89static void mlx5e_update_carrier_work(struct work_struct *work)
90{
91 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
92 update_carrier_work);
93
94 mutex_lock(&priv->state_lock);
95 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
96 mlx5e_update_carrier(priv);
97 mutex_unlock(&priv->state_lock);
98}
99
3947ca18
DJ		 100static void mlx5e_tx_timeout_work(struct work_struct *work)
101{
102 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
103 tx_timeout_work);
104 int err;
105
106 rtnl_lock();
107 mutex_lock(&priv->state_lock);
108 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
109 goto unlock;
110 mlx5e_close_locked(priv->netdev);
111 err = mlx5e_open_locked(priv->netdev);
112 if (err)
113 netdev_err(priv->netdev, "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n",
114 err);
115unlock:
116 mutex_unlock(&priv->state_lock);
117 rtnl_unlock();
118}
119
9218b44d 120static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
f62b8bb8 121{
9218b44d 122 struct mlx5e_sw_stats *s = &priv->stats.sw;
f62b8bb8
AV		 123 struct mlx5e_rq_stats *rq_stats;
124 struct mlx5e_sq_stats *sq_stats;
9218b44d 125 u64 tx_offload_none = 0;
f62b8bb8
AV		 126 int i, j;
127
9218b44d 128 memset(s, 0, sizeof(*s));
f62b8bb8
AV		 129 for (i = 0; i < priv->params.num_channels; i++) {
130 rq_stats = &priv->channel[i]->rq.stats;
131
faf4478b
GP		 132 s->rx_packets += rq_stats->packets;
133 s->rx_bytes += rq_stats->bytes;
bfe6d8d1
GP		 134 s->rx_lro_packets += rq_stats->lro_packets;
135 s->rx_lro_bytes += rq_stats->lro_bytes;
f62b8bb8 136 s->rx_csum_none += rq_stats->csum_none;
bfe6d8d1
GP		 137 s->rx_csum_complete += rq_stats->csum_complete;
138 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
f62b8bb8 139 s->rx_wqe_err += rq_stats->wqe_err;
461017cb 140 s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
54984407 141 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
7219ab34
TT		 142 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
143 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
4415a031
TT		 144 s->rx_cache_reuse += rq_stats->cache_reuse;
145 s->rx_cache_full += rq_stats->cache_full;
146 s->rx_cache_empty += rq_stats->cache_empty;
147 s->rx_cache_busy += rq_stats->cache_busy;
f62b8bb8 148
a4418a6c 149 for (j = 0; j < priv->params.num_tc; j++) {
f62b8bb8
AV		 150 sq_stats = &priv->channel[i]->sq[j].stats;
151
faf4478b
GP		 152 s->tx_packets += sq_stats->packets;
153 s->tx_bytes += sq_stats->bytes;
bfe6d8d1
GP		 154 s->tx_tso_packets += sq_stats->tso_packets;
155 s->tx_tso_bytes += sq_stats->tso_bytes;
156 s->tx_tso_inner_packets += sq_stats->tso_inner_packets;
157 s->tx_tso_inner_bytes += sq_stats->tso_inner_bytes;
f62b8bb8
AV		 158 s->tx_queue_stopped += sq_stats->stopped;
159 s->tx_queue_wake += sq_stats->wake;
160 s->tx_queue_dropped += sq_stats->dropped;
c8cf78fe 161 s->tx_xmit_more += sq_stats->xmit_more;
bfe6d8d1
GP		 162 s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
163 tx_offload_none += sq_stats->csum_none;
f62b8bb8
AV		 164 }
165 }
166
9218b44d 167 /* Update calculated offload counters */
bfe6d8d1
GP		 168 s->tx_csum_partial = s->tx_packets - tx_offload_none - s->tx_csum_partial_inner;
169 s->rx_csum_unnecessary = s->rx_packets - s->rx_csum_none - s->rx_csum_complete;
121fcdc8 170
bfe6d8d1 171 s->link_down_events_phy = MLX5_GET(ppcnt_reg,
121fcdc8
GP		 172 priv->stats.pport.phy_counters,
173 counter_set.phys_layer_cntrs.link_down_events);
9218b44d
GP		 174}
175
176static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
177{
178 int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
179 u32 *out = (u32 *)priv->stats.vport.query_vport_out;
c4f287c4 180 u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
9218b44d
GP		 181 struct mlx5_core_dev *mdev = priv->mdev;
182
f62b8bb8
AV		 183 MLX5_SET(query_vport_counter_in, in, opcode,
184 MLX5_CMD_OP_QUERY_VPORT_COUNTER);
185 MLX5_SET(query_vport_counter_in, in, op_mod, 0);
186 MLX5_SET(query_vport_counter_in, in, other_vport, 0);
187
188 memset(out, 0, outlen);
9218b44d
GP		 189 mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
190}
191
192static void mlx5e_update_pport_counters(struct mlx5e_priv *priv)
193{
194 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
195 struct mlx5_core_dev *mdev = priv->mdev;
196 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
cf678570 197 int prio;
9218b44d
GP		 198 void *out;
199 u32 *in;
200
201 in = mlx5_vzalloc(sz);
202 if (!in)
f62b8bb8
AV		 203 goto free_out;
204
9218b44d 205 MLX5_SET(ppcnt_reg, in, local_port, 1);
f62b8bb8 206
9218b44d
GP		 207 out = pstats->IEEE_802_3_counters;
208 MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
209 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
f62b8bb8 210
9218b44d
GP		 211 out = pstats->RFC_2863_counters;
212 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
213 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
214
215 out = pstats->RFC_2819_counters;
216 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
217 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
593cf338 218
121fcdc8
GP		 219 out = pstats->phy_counters;
220 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
221 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
222
cf678570
GP		 223 MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
224 for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
225 out = pstats->per_prio_counters[prio];
226 MLX5_SET(ppcnt_reg, in, prio_tc, prio);
227 mlx5_core_access_reg(mdev, in, sz, out, sz,
228 MLX5_REG_PPCNT, 0, 0);
229 }
230
f62b8bb8 231free_out:
9218b44d
GP		 232 kvfree(in);
233}
234
235static void mlx5e_update_q_counter(struct mlx5e_priv *priv)
236{
237 struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
238
239 if (!priv->q_counter)
240 return;
241
242 mlx5_core_query_out_of_buffer(priv->mdev, priv->q_counter,
243 &qcnt->rx_out_of_buffer);
244}
245
246void mlx5e_update_stats(struct mlx5e_priv *priv)
247{
9218b44d
GP		 248 mlx5e_update_q_counter(priv);
249 mlx5e_update_vport_counters(priv);
250 mlx5e_update_pport_counters(priv);
121fcdc8 251 mlx5e_update_sw_counters(priv);
f62b8bb8
AV		 252}
253
cb67b832 254void mlx5e_update_stats_work(struct work_struct *work)
f62b8bb8
AV		 255{
256 struct delayed_work *dwork = to_delayed_work(work);
257 struct mlx5e_priv *priv = container_of(dwork, struct mlx5e_priv,
258 update_stats_work);
259 mutex_lock(&priv->state_lock);
260 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
6bfd390b 261 priv->profile->update_stats(priv);
7bb29755
MF		 262 queue_delayed_work(priv->wq, dwork,
263 msecs_to_jiffies(MLX5E_UPDATE_STATS_INTERVAL));
f62b8bb8
AV		 264 }
265 mutex_unlock(&priv->state_lock);
266}
267
daa21560
TT		 268static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
269 enum mlx5_dev_event event, unsigned long param)
f62b8bb8 270{
daa21560
TT		 271 struct mlx5e_priv *priv = vpriv;
272
e0f46eb9 273 if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state))
daa21560
TT		 274 return;
275
f62b8bb8
AV		 276 switch (event) {
277 case MLX5_DEV_EVENT_PORT_UP:
278 case MLX5_DEV_EVENT_PORT_DOWN:
7bb29755 279 queue_work(priv->wq, &priv->update_carrier_work);
f62b8bb8
AV		 280 break;
281
282 default:
283 break;
284 }
285}
286
f62b8bb8
AV		 287static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
288{
e0f46eb9 289 set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
f62b8bb8
AV		 290}
291
292static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
293{
e0f46eb9 294 clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
daa21560 295 synchronize_irq(mlx5_get_msix_vec(priv->mdev, MLX5_EQ_VEC_ASYNC));
f62b8bb8
AV		 296}
297
facc9699
SM		 298#define MLX5E_HW2SW_MTU(hwmtu) (hwmtu - (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
299#define MLX5E_SW2HW_MTU(swmtu) (swmtu + (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
300
7e426671
TT		 301static inline int mlx5e_get_wqe_mtt_sz(void)
302{
303 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
304 * To avoid copying garbage after the mtt array, we allocate
305 * a little more.
306 */
307 return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
308 MLX5_UMR_MTT_ALIGNMENT);
309}
310
311static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, struct mlx5e_sq *sq,
312 struct mlx5e_umr_wqe *wqe, u16 ix)
313{
314 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
315 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
316 struct mlx5_wqe_data_seg *dseg = &wqe->data;
317 struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
318 u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
319 u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
320
321 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
322 ds_cnt);
323 cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
324 cseg->imm = rq->mkey_be;
325
326 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
327 ucseg->klm_octowords =
328 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
329 ucseg->bsf_octowords =
330 cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
331 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
332
333 dseg->lkey = sq->mkey_be;
334 dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
335}
336
337static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
338 struct mlx5e_channel *c)
339{
340 int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
341 int mtt_sz = mlx5e_get_wqe_mtt_sz();
342 int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
343 int i;
344
345 rq->wqe_info = kzalloc_node(wq_sz * sizeof(*rq->wqe_info),
346 GFP_KERNEL, cpu_to_node(c->cpu));
347 if (!rq->wqe_info)
348 goto err_out;
349
350 /* We allocate more than mtt_sz as we will align the pointer */
351 rq->mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
352 cpu_to_node(c->cpu));
353 if (unlikely(!rq->mtt_no_align))
354 goto err_free_wqe_info;
355
356 for (i = 0; i < wq_sz; i++) {
357 struct mlx5e_mpw_info *wi = &rq->wqe_info[i];
358
359 wi->umr.mtt = PTR_ALIGN(rq->mtt_no_align + i * mtt_alloc,
360 MLX5_UMR_ALIGN);
361 wi->umr.mtt_addr = dma_map_single(c->pdev, wi->umr.mtt, mtt_sz,
362 PCI_DMA_TODEVICE);
363 if (unlikely(dma_mapping_error(c->pdev, wi->umr.mtt_addr)))
364 goto err_unmap_mtts;
365
366 mlx5e_build_umr_wqe(rq, &c->icosq, &wi->umr.wqe, i);
367 }
368
369 return 0;
370
371err_unmap_mtts:
372 while (--i >= 0) {
373 struct mlx5e_mpw_info *wi = &rq->wqe_info[i];
374
375 dma_unmap_single(c->pdev, wi->umr.mtt_addr, mtt_sz,
376 PCI_DMA_TODEVICE);
377 }
378 kfree(rq->mtt_no_align);
379err_free_wqe_info:
380 kfree(rq->wqe_info);
381
382err_out:
383 return -ENOMEM;
384}
385
386static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
387{
388 int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
389 int mtt_sz = mlx5e_get_wqe_mtt_sz();
390 int i;
391
392 for (i = 0; i < wq_sz; i++) {
393 struct mlx5e_mpw_info *wi = &rq->wqe_info[i];
394
395 dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz,
396 PCI_DMA_TODEVICE);
397 }
398 kfree(rq->mtt_no_align);
399 kfree(rq->wqe_info);
400}
401
f62b8bb8
AV		 402static int mlx5e_create_rq(struct mlx5e_channel *c,
403 struct mlx5e_rq_param *param,
404 struct mlx5e_rq *rq)
405{
406 struct mlx5e_priv *priv = c->priv;
407 struct mlx5_core_dev *mdev = priv->mdev;
408 void *rqc = param->rqc;
409 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
461017cb 410 u32 byte_count;
f62b8bb8
AV		 411 int wq_sz;
412 int err;
413 int i;
414
311c7c71
SM		 415 param->wq.db_numa_node = cpu_to_node(c->cpu);
416
f62b8bb8
AV		 417 err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
418 &rq->wq_ctrl);
419 if (err)
420 return err;
421
422 rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
423
424 wq_sz = mlx5_wq_ll_get_size(&rq->wq);
f62b8bb8 425
7e426671
TT		 426 rq->wq_type = priv->params.rq_wq_type;
427 rq->pdev = c->pdev;
428 rq->netdev = c->netdev;
429 rq->tstamp = &priv->tstamp;
430 rq->channel = c;
431 rq->ix = c->ix;
432 rq->priv = c->priv;
433
461017cb
TT		 434 switch (priv->params.rq_wq_type) {
435 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
461017cb
TT		 436 rq->handle_rx_cqe = mlx5e_handle_rx_cqe_mpwrq;
437 rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
6cd392a0 438 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
461017cb 439
fe4c988b
SM		 440 rq->mpwqe_mtt_offset = c->ix *
441 MLX5E_REQUIRED_MTTS(1, BIT(priv->params.log_rq_size));
442
d9d9f156
TT		 443 rq->mpwqe_stride_sz = BIT(priv->params.mpwqe_log_stride_sz);
444 rq->mpwqe_num_strides = BIT(priv->params.mpwqe_log_num_strides);
445 rq->wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
461017cb 446 byte_count = rq->wqe_sz;
7e426671
TT		 447 rq->mkey_be = cpu_to_be32(c->priv->umr_mkey.key);
448 err = mlx5e_rq_alloc_mpwqe_info(rq, c);
449 if (err)
450 goto err_rq_wq_destroy;
461017cb
TT		 451 break;
452 default: /* MLX5_WQ_TYPE_LINKED_LIST */
453 rq->skb = kzalloc_node(wq_sz * sizeof(*rq->skb), GFP_KERNEL,
454 cpu_to_node(c->cpu));
455 if (!rq->skb) {
456 err = -ENOMEM;
457 goto err_rq_wq_destroy;
458 }
459 rq->handle_rx_cqe = mlx5e_handle_rx_cqe;
460 rq->alloc_wqe = mlx5e_alloc_rx_wqe;
6cd392a0 461 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
461017cb
TT		 462
463 rq->wqe_sz = (priv->params.lro_en) ?
464 priv->params.lro_wqe_sz :
465 MLX5E_SW2HW_MTU(priv->netdev->mtu);
c5adb96f
TT		 466 rq->wqe_sz = SKB_DATA_ALIGN(rq->wqe_sz);
467 byte_count = rq->wqe_sz;
461017cb 468 byte_count |= MLX5_HW_START_PADDING;
7e426671 469 rq->mkey_be = c->mkey_be;
461017cb 470 }
f62b8bb8
AV		 471
472 for (i = 0; i < wq_sz; i++) {
473 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
474
461017cb 475 wqe->data.byte_count = cpu_to_be32(byte_count);
7e426671 476 wqe->data.lkey = rq->mkey_be;
f62b8bb8
AV		 477 }
478
cb3c7fd4
GR		 479 INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
480 rq->am.mode = priv->params.rx_cq_period_mode;
481
4415a031
TT		 482 rq->page_cache.head = 0;
483 rq->page_cache.tail = 0;
484
f62b8bb8
AV		 485 return 0;
486
487err_rq_wq_destroy:
488 mlx5_wq_destroy(&rq->wq_ctrl);
489
490 return err;
491}
492
493static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
494{
4415a031
TT		 495 int i;
496
461017cb
TT		 497 switch (rq->wq_type) {
498 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
7e426671 499 mlx5e_rq_free_mpwqe_info(rq);
461017cb
TT		 500 break;
501 default: /* MLX5_WQ_TYPE_LINKED_LIST */
502 kfree(rq->skb);
503 }
504
4415a031
TT		 505 for (i = rq->page_cache.head; i != rq->page_cache.tail;
506 i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
507 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
508
509 mlx5e_page_release(rq, dma_info, false);
510 }
f62b8bb8
AV		 511 mlx5_wq_destroy(&rq->wq_ctrl);
512}
513
514static int mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
515{
50cfa25a 516 struct mlx5e_priv *priv = rq->priv;
f62b8bb8
AV		 517 struct mlx5_core_dev *mdev = priv->mdev;
518
519 void *in;
520 void *rqc;
521 void *wq;
522 int inlen;
523 int err;
524
525 inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
526 sizeof(u64) * rq->wq_ctrl.buf.npages;
527 in = mlx5_vzalloc(inlen);
528 if (!in)
529 return -ENOMEM;
530
531 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
532 wq = MLX5_ADDR_OF(rqc, rqc, wq);
533
534 memcpy(rqc, param->rqc, sizeof(param->rqc));
535
97de9f31 536 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn);
f62b8bb8 537 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
36350114 538 MLX5_SET(rqc, rqc, vsd, priv->params.vlan_strip_disable);
f62b8bb8 539 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
68cdf5d6 540 MLX5_ADAPTER_PAGE_SHIFT);
f62b8bb8
AV		 541 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
542
543 mlx5_fill_page_array(&rq->wq_ctrl.buf,
544 (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
545
7db22ffb 546 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
f62b8bb8
AV		 547
548 kvfree(in);
549
550 return err;
551}
552
36350114
GP		 553static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
554 int next_state)
f62b8bb8
AV		 555{
556 struct mlx5e_channel *c = rq->channel;
557 struct mlx5e_priv *priv = c->priv;
558 struct mlx5_core_dev *mdev = priv->mdev;
559
560 void *in;
561 void *rqc;
562 int inlen;
563 int err;
564
565 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
566 in = mlx5_vzalloc(inlen);
567 if (!in)
568 return -ENOMEM;
569
570 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
571
572 MLX5_SET(modify_rq_in, in, rq_state, curr_state);
573 MLX5_SET(rqc, rqc, state, next_state);
574
7db22ffb 575 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
f62b8bb8
AV		 576
577 kvfree(in);
578
579 return err;
580}
581
36350114
GP		 582static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
583{
584 struct mlx5e_channel *c = rq->channel;
585 struct mlx5e_priv *priv = c->priv;
586 struct mlx5_core_dev *mdev = priv->mdev;
587
588 void *in;
589 void *rqc;
590 int inlen;
591 int err;
592
593 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
594 in = mlx5_vzalloc(inlen);
595 if (!in)
596 return -ENOMEM;
597
598 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
599
600 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
83b502a1
AV		 601 MLX5_SET64(modify_rq_in, in, modify_bitmask,
602 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
36350114
GP		 603 MLX5_SET(rqc, rqc, vsd, vsd);
604 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
605
606 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
607
608 kvfree(in);
609
610 return err;
611}
612
f62b8bb8
AV		 613static void mlx5e_disable_rq(struct mlx5e_rq *rq)
614{
50cfa25a 615 mlx5_core_destroy_rq(rq->priv->mdev, rq->rqn);
f62b8bb8
AV		 616}
617
618static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
619{
01c196a2 620 unsigned long exp_time = jiffies + msecs_to_jiffies(20000);
f62b8bb8
AV		 621 struct mlx5e_channel *c = rq->channel;
622 struct mlx5e_priv *priv = c->priv;
623 struct mlx5_wq_ll *wq = &rq->wq;
f62b8bb8 624
01c196a2 625 while (time_before(jiffies, exp_time)) {
f62b8bb8
AV		 626 if (wq->cur_sz >= priv->params.min_rx_wqes)
627 return 0;
628
629 msleep(20);
630 }
631
632 return -ETIMEDOUT;
633}
634
f2fde18c
SM		 635static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
636{
637 struct mlx5_wq_ll *wq = &rq->wq;
638 struct mlx5e_rx_wqe *wqe;
639 __be16 wqe_ix_be;
640 u16 wqe_ix;
641
8484f9ed
SM		 642 /* UMR WQE (if in progress) is always at wq->head */
643 if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
7e426671 644 mlx5e_free_rx_mpwqe(rq, &rq->wqe_info[wq->head]);
8484f9ed 645
f2fde18c
SM		 646 while (!mlx5_wq_ll_is_empty(wq)) {
647 wqe_ix_be = *wq->tail_next;
648 wqe_ix = be16_to_cpu(wqe_ix_be);
649 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_ix);
650 rq->dealloc_wqe(rq, wqe_ix);
651 mlx5_wq_ll_pop(&rq->wq, wqe_ix_be,
652 &wqe->next.next_wqe_index);
653 }
654}
655
f62b8bb8
AV		 656static int mlx5e_open_rq(struct mlx5e_channel *c,
657 struct mlx5e_rq_param *param,
658 struct mlx5e_rq *rq)
659{
d3c9bc27
TT		 660 struct mlx5e_sq *sq = &c->icosq;
661 u16 pi = sq->pc & sq->wq.sz_m1;
f62b8bb8
AV		 662 int err;
663
664 err = mlx5e_create_rq(c, param, rq);
665 if (err)
666 return err;
667
668 err = mlx5e_enable_rq(rq, param);
669 if (err)
670 goto err_destroy_rq;
671
36350114 672 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
f62b8bb8
AV		 673 if (err)
674 goto err_disable_rq;
675
cb3c7fd4
GR		 676 if (param->am_enabled)
677 set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
678
d3c9bc27
TT		 679 sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_NOP;
680 sq->ico_wqe_info[pi].num_wqebbs = 1;
681 mlx5e_send_nop(sq, true); /* trigger mlx5e_post_rx_wqes() */
f62b8bb8
AV		 682
683 return 0;
684
685err_disable_rq:
686 mlx5e_disable_rq(rq);
687err_destroy_rq:
688 mlx5e_destroy_rq(rq);
689
690 return err;
691}
692
693static void mlx5e_close_rq(struct mlx5e_rq *rq)
694{
f2fde18c 695 set_bit(MLX5E_RQ_STATE_FLUSH, &rq->state);
f62b8bb8 696 napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
cb3c7fd4
GR		 697 cancel_work_sync(&rq->am.work);
698
f62b8bb8 699 mlx5e_disable_rq(rq);
6cd392a0 700 mlx5e_free_rx_descs(rq);
f62b8bb8
AV		 701 mlx5e_destroy_rq(rq);
702}
703
704static void mlx5e_free_sq_db(struct mlx5e_sq *sq)
705{
34802a42 706 kfree(sq->wqe_info);
f62b8bb8
AV		 707 kfree(sq->dma_fifo);
708 kfree(sq->skb);
709}
710
711static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)
712{
713 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
714 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
715
716 sq->skb = kzalloc_node(wq_sz * sizeof(*sq->skb), GFP_KERNEL, numa);
717 sq->dma_fifo = kzalloc_node(df_sz * sizeof(*sq->dma_fifo), GFP_KERNEL,
718 numa);
34802a42
AS		 719 sq->wqe_info = kzalloc_node(wq_sz * sizeof(*sq->wqe_info), GFP_KERNEL,
720 numa);
f62b8bb8 721
34802a42 722 if (!sq->skb || !sq->dma_fifo || !sq->wqe_info) {
f62b8bb8
AV		 723 mlx5e_free_sq_db(sq);
724 return -ENOMEM;
725 }
726
727 sq->dma_fifo_mask = df_sz - 1;
728
729 return 0;
730}
731
732static int mlx5e_create_sq(struct mlx5e_channel *c,
733 int tc,
734 struct mlx5e_sq_param *param,
735 struct mlx5e_sq *sq)
736{
737 struct mlx5e_priv *priv = c->priv;
738 struct mlx5_core_dev *mdev = priv->mdev;
739
740 void *sqc = param->sqc;
741 void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
742 int err;
743
fd4782c2 744 err = mlx5_alloc_map_uar(mdev, &sq->uar, !!MLX5_CAP_GEN(mdev, bf));
f62b8bb8
AV		 745 if (err)
746 return err;
747
311c7c71
SM		 748 param->wq.db_numa_node = cpu_to_node(c->cpu);
749
f62b8bb8
AV		 750 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
751 &sq->wq_ctrl);
752 if (err)
753 goto err_unmap_free_uar;
754
755 sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
0ba42241
ML		 756 if (sq->uar.bf_map) {
757 set_bit(MLX5E_SQ_STATE_BF_ENABLE, &sq->state);
758 sq->uar_map = sq->uar.bf_map;
759 } else {
760 sq->uar_map = sq->uar.map;
761 }
f62b8bb8 762 sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
58d52291 763 sq->max_inline = param->max_inline;
cff92d7c
HHZ		 764 sq->min_inline_mode =
765 MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5E_INLINE_MODE_VPORT_CONTEXT ?
766 param->min_inline_mode : 0;
f62b8bb8 767
7ec0bb22
DC		 768 err = mlx5e_alloc_sq_db(sq, cpu_to_node(c->cpu));
769 if (err)
f62b8bb8
AV		 770 goto err_sq_wq_destroy;
771
d3c9bc27
TT		 772 if (param->icosq) {
773 u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
774
775 sq->ico_wqe_info = kzalloc_node(sizeof(*sq->ico_wqe_info) *
776 wq_sz,
777 GFP_KERNEL,
778 cpu_to_node(c->cpu));
779 if (!sq->ico_wqe_info) {
780 err = -ENOMEM;
781 goto err_free_sq_db;
782 }
783 } else {
784 int txq_ix;
785
786 txq_ix = c->ix + tc * priv->params.num_channels;
787 sq->txq = netdev_get_tx_queue(priv->netdev, txq_ix);
788 priv->txq_to_sq_map[txq_ix] = sq;
789 }
f62b8bb8 790
88a85f99 791 sq->pdev = c->pdev;
ef9814de 792 sq->tstamp = &priv->tstamp;
88a85f99
AS		 793 sq->mkey_be = c->mkey_be;
794 sq->channel = c;
795 sq->tc = tc;
796 sq->edge = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;
797 sq->bf_budget = MLX5E_SQ_BF_BUDGET;
f62b8bb8
AV		 798
799 return 0;
800
d3c9bc27
TT		 801err_free_sq_db:
802 mlx5e_free_sq_db(sq);
803
f62b8bb8
AV		 804err_sq_wq_destroy:
805 mlx5_wq_destroy(&sq->wq_ctrl);
806
807err_unmap_free_uar:
808 mlx5_unmap_free_uar(mdev, &sq->uar);
809
810 return err;
811}
812
813static void mlx5e_destroy_sq(struct mlx5e_sq *sq)
814{
815 struct mlx5e_channel *c = sq->channel;
816 struct mlx5e_priv *priv = c->priv;
817
d3c9bc27 818 kfree(sq->ico_wqe_info);
f62b8bb8
AV		 819 mlx5e_free_sq_db(sq);
820 mlx5_wq_destroy(&sq->wq_ctrl);
821 mlx5_unmap_free_uar(priv->mdev, &sq->uar);
822}
823
824static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
825{
826 struct mlx5e_channel *c = sq->channel;
827 struct mlx5e_priv *priv = c->priv;
828 struct mlx5_core_dev *mdev = priv->mdev;
829
830 void *in;
831 void *sqc;
832 void *wq;
833 int inlen;
834 int err;
835
836 inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
837 sizeof(u64) * sq->wq_ctrl.buf.npages;
838 in = mlx5_vzalloc(inlen);
839 if (!in)
840 return -ENOMEM;
841
842 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
843 wq = MLX5_ADDR_OF(sqc, sqc, wq);
844
845 memcpy(sqc, param->sqc, sizeof(param->sqc));
846
d3c9bc27
TT		 847 MLX5_SET(sqc, sqc, tis_num_0, param->icosq ? 0 : priv->tisn[sq->tc]);
848 MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
cff92d7c 849 MLX5_SET(sqc, sqc, min_wqe_inline_mode, sq->min_inline_mode);
f62b8bb8 850 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
d3c9bc27 851 MLX5_SET(sqc, sqc, tis_lst_sz, param->icosq ? 0 : 1);
f62b8bb8
AV		 852 MLX5_SET(sqc, sqc, flush_in_error_en, 1);
853
854 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
855 MLX5_SET(wq, wq, uar_page, sq->uar.index);
856 MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
68cdf5d6 857 MLX5_ADAPTER_PAGE_SHIFT);
f62b8bb8
AV		 858 MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
859
860 mlx5_fill_page_array(&sq->wq_ctrl.buf,
861 (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
862
7db22ffb 863 err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
f62b8bb8
AV		 864
865 kvfree(in);
866
867 return err;
868}
869
507f0c81
YP		 870static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state,
871 int next_state, bool update_rl, int rl_index)
f62b8bb8
AV		 872{
873 struct mlx5e_channel *c = sq->channel;
874 struct mlx5e_priv *priv = c->priv;
875 struct mlx5_core_dev *mdev = priv->mdev;
876
877 void *in;
878 void *sqc;
879 int inlen;
880 int err;
881
882 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
883 in = mlx5_vzalloc(inlen);
884 if (!in)
885 return -ENOMEM;
886
887 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
888
889 MLX5_SET(modify_sq_in, in, sq_state, curr_state);
890 MLX5_SET(sqc, sqc, state, next_state);
507f0c81
YP		 891 if (update_rl && next_state == MLX5_SQC_STATE_RDY) {
892 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
893 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
894 }
f62b8bb8 895
7db22ffb 896 err = mlx5_core_modify_sq(mdev, sq->sqn, in, inlen);
f62b8bb8
AV		 897
898 kvfree(in);
899
900 return err;
901}
902
903static void mlx5e_disable_sq(struct mlx5e_sq *sq)
904{
905 struct mlx5e_channel *c = sq->channel;
906 struct mlx5e_priv *priv = c->priv;
907 struct mlx5_core_dev *mdev = priv->mdev;
908
7db22ffb 909 mlx5_core_destroy_sq(mdev, sq->sqn);
507f0c81
YP		 910 if (sq->rate_limit)
911 mlx5_rl_remove_rate(mdev, sq->rate_limit);
f62b8bb8
AV		 912}
913
914static int mlx5e_open_sq(struct mlx5e_channel *c,
915 int tc,
916 struct mlx5e_sq_param *param,
917 struct mlx5e_sq *sq)
918{
919 int err;
920
921 err = mlx5e_create_sq(c, tc, param, sq);
922 if (err)
923 return err;
924
925 err = mlx5e_enable_sq(sq, param);
926 if (err)
927 goto err_destroy_sq;
928
507f0c81
YP		 929 err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
930 false, 0);
f62b8bb8
AV		 931 if (err)
932 goto err_disable_sq;
933
d3c9bc27 934 if (sq->txq) {
d3c9bc27
TT		 935 netdev_tx_reset_queue(sq->txq);
936 netif_tx_start_queue(sq->txq);
937 }
f62b8bb8
AV		 938
939 return 0;
940
941err_disable_sq:
942 mlx5e_disable_sq(sq);
943err_destroy_sq:
944 mlx5e_destroy_sq(sq);
945
946 return err;
947}
948
949static inline void netif_tx_disable_queue(struct netdev_queue *txq)
950{
951 __netif_tx_lock_bh(txq);
952 netif_tx_stop_queue(txq);
953 __netif_tx_unlock_bh(txq);
954}
955
956static void mlx5e_close_sq(struct mlx5e_sq *sq)
957{
6e8dd6d6
SM		 958 set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
959 /* prevent netif_tx_wake_queue */
960 napi_synchronize(&sq->channel->napi);
29429f33 961
d3c9bc27 962 if (sq->txq) {
d3c9bc27 963 netif_tx_disable_queue(sq->txq);
f62b8bb8 964
6e8dd6d6 965 /* last doorbell out, godspeed .. */
d3c9bc27
TT		 966 if (mlx5e_sq_has_room_for(sq, 1))
967 mlx5e_send_nop(sq, true);
29429f33 968 }
f62b8bb8 969
f62b8bb8 970 mlx5e_disable_sq(sq);
6e8dd6d6 971 mlx5e_free_tx_descs(sq);
f62b8bb8
AV		 972 mlx5e_destroy_sq(sq);
973}
974
975static int mlx5e_create_cq(struct mlx5e_channel *c,
976 struct mlx5e_cq_param *param,
977 struct mlx5e_cq *cq)
978{
979 struct mlx5e_priv *priv = c->priv;
980 struct mlx5_core_dev *mdev = priv->mdev;
981 struct mlx5_core_cq *mcq = &cq->mcq;
982 int eqn_not_used;
0b6e26ce 983 unsigned int irqn;
f62b8bb8
AV		 984 int err;
985 u32 i;
986
311c7c71
SM		 987 param->wq.buf_numa_node = cpu_to_node(c->cpu);
988 param->wq.db_numa_node = cpu_to_node(c->cpu);
f62b8bb8
AV		 989 param->eq_ix = c->ix;
990
991 err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
992 &cq->wq_ctrl);
993 if (err)
994 return err;
995
996 mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
997
998 cq->napi = &c->napi;
999
1000 mcq->cqe_sz = 64;
1001 mcq->set_ci_db = cq->wq_ctrl.db.db;
1002 mcq->arm_db = cq->wq_ctrl.db.db + 1;
1003 *mcq->set_ci_db = 0;
1004 *mcq->arm_db = 0;
1005 mcq->vector = param->eq_ix;
1006 mcq->comp = mlx5e_completion_event;
1007 mcq->event = mlx5e_cq_error_event;
1008 mcq->irqn = irqn;
b50d292b 1009 mcq->uar = &mdev->mlx5e_res.cq_uar;
f62b8bb8
AV		 1010
1011 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1012 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1013
1014 cqe->op_own = 0xf1;
1015 }
1016
1017 cq->channel = c;
50cfa25a 1018 cq->priv = priv;
f62b8bb8
AV		 1019
1020 return 0;
1021}
1022
1023static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1024{
1025 mlx5_wq_destroy(&cq->wq_ctrl);
1026}
1027
1028static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1029{
50cfa25a 1030 struct mlx5e_priv *priv = cq->priv;
f62b8bb8
AV		 1031 struct mlx5_core_dev *mdev = priv->mdev;
1032 struct mlx5_core_cq *mcq = &cq->mcq;
1033
1034 void *in;
1035 void *cqc;
1036 int inlen;
0b6e26ce 1037 unsigned int irqn_not_used;
f62b8bb8
AV		 1038 int eqn;
1039 int err;
1040
1041 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1042 sizeof(u64) * cq->wq_ctrl.buf.npages;
1043 in = mlx5_vzalloc(inlen);
1044 if (!in)
1045 return -ENOMEM;
1046
1047 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1048
1049 memcpy(cqc, param->cqc, sizeof(param->cqc));
1050
1051 mlx5_fill_page_array(&cq->wq_ctrl.buf,
1052 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1053
1054 mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1055
9908aa29 1056 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode);
f62b8bb8
AV		 1057 MLX5_SET(cqc, cqc, c_eqn, eqn);
1058 MLX5_SET(cqc, cqc, uar_page, mcq->uar->index);
1059 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
68cdf5d6 1060 MLX5_ADAPTER_PAGE_SHIFT);
f62b8bb8
AV		 1061 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
1062
1063 err = mlx5_core_create_cq(mdev, mcq, in, inlen);
1064
1065 kvfree(in);
1066
1067 if (err)
1068 return err;
1069
1070 mlx5e_cq_arm(cq);
1071
1072 return 0;
1073}
1074
1075static void mlx5e_disable_cq(struct mlx5e_cq *cq)
1076{
50cfa25a 1077 struct mlx5e_priv *priv = cq->priv;
f62b8bb8
AV		 1078 struct mlx5_core_dev *mdev = priv->mdev;
1079
1080 mlx5_core_destroy_cq(mdev, &cq->mcq);
1081}
1082
1083static int mlx5e_open_cq(struct mlx5e_channel *c,
1084 struct mlx5e_cq_param *param,
1085 struct mlx5e_cq *cq,
9908aa29 1086 struct mlx5e_cq_moder moderation)
f62b8bb8
AV		 1087{
1088 int err;
1089 struct mlx5e_priv *priv = c->priv;
1090 struct mlx5_core_dev *mdev = priv->mdev;
1091
1092 err = mlx5e_create_cq(c, param, cq);
1093 if (err)
1094 return err;
1095
1096 err = mlx5e_enable_cq(cq, param);
1097 if (err)
1098 goto err_destroy_cq;
1099
7524a5d8
GP		 1100 if (MLX5_CAP_GEN(mdev, cq_moderation))
1101 mlx5_core_modify_cq_moderation(mdev, &cq->mcq,
9908aa29
TT		 1102 moderation.usec,
1103 moderation.pkts);
f62b8bb8
AV		 1104 return 0;
1105
1106err_destroy_cq:
1107 mlx5e_destroy_cq(cq);
1108
1109 return err;
1110}
1111
1112static void mlx5e_close_cq(struct mlx5e_cq *cq)
1113{
1114 mlx5e_disable_cq(cq);
1115 mlx5e_destroy_cq(cq);
1116}
1117
1118static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
1119{
1120 return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
1121}
1122
1123static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1124 struct mlx5e_channel_param *cparam)
1125{
1126 struct mlx5e_priv *priv = c->priv;
1127 int err;
1128 int tc;
1129
1130 for (tc = 0; tc < c->num_tc; tc++) {
1131 err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
9908aa29 1132 priv->params.tx_cq_moderation);
f62b8bb8
AV		 1133 if (err)
1134 goto err_close_tx_cqs;
f62b8bb8
AV		 1135 }
1136
1137 return 0;
1138
1139err_close_tx_cqs:
1140 for (tc--; tc >= 0; tc--)
1141 mlx5e_close_cq(&c->sq[tc].cq);
1142
1143 return err;
1144}
1145
1146static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1147{
1148 int tc;
1149
1150 for (tc = 0; tc < c->num_tc; tc++)
1151 mlx5e_close_cq(&c->sq[tc].cq);
1152}
1153
1154static int mlx5e_open_sqs(struct mlx5e_channel *c,
1155 struct mlx5e_channel_param *cparam)
1156{
1157 int err;
1158 int tc;
1159
1160 for (tc = 0; tc < c->num_tc; tc++) {
1161 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
1162 if (err)
1163 goto err_close_sqs;
1164 }
1165
1166 return 0;
1167
1168err_close_sqs:
1169 for (tc--; tc >= 0; tc--)
1170 mlx5e_close_sq(&c->sq[tc]);
1171
1172 return err;
1173}
1174
1175static void mlx5e_close_sqs(struct mlx5e_channel *c)
1176{
1177 int tc;
1178
1179 for (tc = 0; tc < c->num_tc; tc++)
1180 mlx5e_close_sq(&c->sq[tc]);
1181}
1182
5283af89 1183static void mlx5e_build_channeltc_to_txq_map(struct mlx5e_priv *priv, int ix)
03289b88
SM		 1184{
1185 int i;
1186
6bfd390b 1187 for (i = 0; i < priv->profile->max_tc; i++)
5283af89
RS		 1188 priv->channeltc_to_txq_map[ix][i] =
1189 ix + i * priv->params.num_channels;
03289b88
SM		 1190}
1191
507f0c81
YP		 1192static int mlx5e_set_sq_maxrate(struct net_device *dev,
1193 struct mlx5e_sq *sq, u32 rate)
1194{
1195 struct mlx5e_priv *priv = netdev_priv(dev);
1196 struct mlx5_core_dev *mdev = priv->mdev;
1197 u16 rl_index = 0;
1198 int err;
1199
1200 if (rate == sq->rate_limit)
1201 /* nothing to do */
1202 return 0;
1203
1204 if (sq->rate_limit)
1205 /* remove current rl index to free space to next ones */
1206 mlx5_rl_remove_rate(mdev, sq->rate_limit);
1207
1208 sq->rate_limit = 0;
1209
1210 if (rate) {
1211 err = mlx5_rl_add_rate(mdev, rate, &rl_index);
1212 if (err) {
1213 netdev_err(dev, "Failed configuring rate %u: %d\n",
1214 rate, err);
1215 return err;
1216 }
1217 }
1218
1219 err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY,
1220 MLX5_SQC_STATE_RDY, true, rl_index);
1221 if (err) {
1222 netdev_err(dev, "Failed configuring rate %u: %d\n",
1223 rate, err);
1224 /* remove the rate from the table */
1225 if (rate)
1226 mlx5_rl_remove_rate(mdev, rate);
1227 return err;
1228 }
1229
1230 sq->rate_limit = rate;
1231 return 0;
1232}
1233
1234static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1235{
1236 struct mlx5e_priv *priv = netdev_priv(dev);
1237 struct mlx5_core_dev *mdev = priv->mdev;
1238 struct mlx5e_sq *sq = priv->txq_to_sq_map[index];
1239 int err = 0;
1240
1241 if (!mlx5_rl_is_supported(mdev)) {
1242 netdev_err(dev, "Rate limiting is not supported on this device\n");
1243 return -EINVAL;
1244 }
1245
1246 /* rate is given in Mb/sec, HW config is in Kb/sec */
1247 rate = rate << 10;
1248
1249 /* Check whether rate in valid range, 0 is always valid */
1250 if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1251 netdev_err(dev, "TX rate %u, is not in range\n", rate);
1252 return -ERANGE;
1253 }
1254
1255 mutex_lock(&priv->state_lock);
1256 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1257 err = mlx5e_set_sq_maxrate(dev, sq, rate);
1258 if (!err)
1259 priv->tx_rates[index] = rate;
1260 mutex_unlock(&priv->state_lock);
1261
1262 return err;
1263}
1264
f62b8bb8
AV		 1265static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1266 struct mlx5e_channel_param *cparam,
1267 struct mlx5e_channel **cp)
1268{
9908aa29 1269 struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
f62b8bb8 1270 struct net_device *netdev = priv->netdev;
cb3c7fd4 1271 struct mlx5e_cq_moder rx_cq_profile;
f62b8bb8
AV		 1272 int cpu = mlx5e_get_cpu(priv, ix);
1273 struct mlx5e_channel *c;
507f0c81 1274 struct mlx5e_sq *sq;
f62b8bb8 1275 int err;
507f0c81 1276 int i;
f62b8bb8
AV		 1277
1278 c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1279 if (!c)
1280 return -ENOMEM;
1281
1282 c->priv = priv;
1283 c->ix = ix;
1284 c->cpu = cpu;
1285 c->pdev = &priv->mdev->pdev->dev;
1286 c->netdev = priv->netdev;
b50d292b 1287 c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
a4418a6c 1288 c->num_tc = priv->params.num_tc;
f62b8bb8 1289
cb3c7fd4
GR		 1290 if (priv->params.rx_am_enabled)
1291 rx_cq_profile = mlx5e_am_get_def_profile(priv->params.rx_cq_period_mode);
1292 else
1293 rx_cq_profile = priv->params.rx_cq_moderation;
1294
5283af89 1295 mlx5e_build_channeltc_to_txq_map(priv, ix);
03289b88 1296
f62b8bb8
AV		 1297 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1298
9908aa29 1299 err = mlx5e_open_cq(c, &cparam->icosq_cq, &c->icosq.cq, icosq_cq_moder);
f62b8bb8
AV		 1300 if (err)
1301 goto err_napi_del;
1302
d3c9bc27
TT		 1303 err = mlx5e_open_tx_cqs(c, cparam);
1304 if (err)
1305 goto err_close_icosq_cq;
1306
f62b8bb8 1307 err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
cb3c7fd4 1308 rx_cq_profile);
f62b8bb8
AV		 1309 if (err)
1310 goto err_close_tx_cqs;
f62b8bb8
AV		 1311
1312 napi_enable(&c->napi);
1313
d3c9bc27 1314 err = mlx5e_open_sq(c, 0, &cparam->icosq, &c->icosq);
f62b8bb8
AV		 1315 if (err)
1316 goto err_disable_napi;
1317
d3c9bc27
TT		 1318 err = mlx5e_open_sqs(c, cparam);
1319 if (err)
1320 goto err_close_icosq;
1321
507f0c81
YP		 1322 for (i = 0; i < priv->params.num_tc; i++) {
1323 u32 txq_ix = priv->channeltc_to_txq_map[ix][i];
1324
1325 if (priv->tx_rates[txq_ix]) {
1326 sq = priv->txq_to_sq_map[txq_ix];
1327 mlx5e_set_sq_maxrate(priv->netdev, sq,
1328 priv->tx_rates[txq_ix]);
1329 }
1330 }
1331
f62b8bb8
AV		 1332 err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
1333 if (err)
1334 goto err_close_sqs;
1335
1336 netif_set_xps_queue(netdev, get_cpu_mask(c->cpu), ix);
1337 *cp = c;
1338
1339 return 0;
1340
1341err_close_sqs:
1342 mlx5e_close_sqs(c);
1343
d3c9bc27
TT		 1344err_close_icosq:
1345 mlx5e_close_sq(&c->icosq);
1346
f62b8bb8
AV		 1347err_disable_napi:
1348 napi_disable(&c->napi);
1349 mlx5e_close_cq(&c->rq.cq);
1350
1351err_close_tx_cqs:
1352 mlx5e_close_tx_cqs(c);
1353
d3c9bc27
TT		 1354err_close_icosq_cq:
1355 mlx5e_close_cq(&c->icosq.cq);
1356
f62b8bb8
AV		 1357err_napi_del:
1358 netif_napi_del(&c->napi);
7ae92ae5 1359 napi_hash_del(&c->napi);
f62b8bb8
AV		 1360 kfree(c);
1361
1362 return err;
1363}
1364
1365static void mlx5e_close_channel(struct mlx5e_channel *c)
1366{
1367 mlx5e_close_rq(&c->rq);
1368 mlx5e_close_sqs(c);
d3c9bc27 1369 mlx5e_close_sq(&c->icosq);
f62b8bb8
AV		 1370 napi_disable(&c->napi);
1371 mlx5e_close_cq(&c->rq.cq);
1372 mlx5e_close_tx_cqs(c);
d3c9bc27 1373 mlx5e_close_cq(&c->icosq.cq);
f62b8bb8 1374 netif_napi_del(&c->napi);
7ae92ae5
ED		 1375
1376 napi_hash_del(&c->napi);
1377 synchronize_rcu();
1378
f62b8bb8
AV		 1379 kfree(c);
1380}
1381
1382static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
1383 struct mlx5e_rq_param *param)
1384{
1385 void *rqc = param->rqc;
1386 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1387
461017cb
TT		 1388 switch (priv->params.rq_wq_type) {
1389 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1390 MLX5_SET(wq, wq, log_wqe_num_of_strides,
d9d9f156 1391 priv->params.mpwqe_log_num_strides - 9);
461017cb 1392 MLX5_SET(wq, wq, log_wqe_stride_size,
d9d9f156 1393 priv->params.mpwqe_log_stride_sz - 6);
461017cb
TT		 1394 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
1395 break;
1396 default: /* MLX5_WQ_TYPE_LINKED_LIST */
1397 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1398 }
1399
f62b8bb8
AV		 1400 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1401 MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe)));
1402 MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
b50d292b 1403 MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn);
593cf338 1404 MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
f62b8bb8 1405
311c7c71 1406 param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
f62b8bb8 1407 param->wq.linear = 1;
cb3c7fd4
GR		 1408
1409 param->am_enabled = priv->params.rx_am_enabled;
f62b8bb8
AV		 1410}
1411
556dd1b9
TT		 1412static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
1413{
1414 void *rqc = param->rqc;
1415 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1416
1417 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1418 MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe)));
1419}
1420
d3c9bc27
TT		 1421static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
1422 struct mlx5e_sq_param *param)
f62b8bb8
AV		 1423{
1424 void *sqc = param->sqc;
1425 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1426
f62b8bb8 1427 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
b50d292b 1428 MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn);
f62b8bb8 1429
311c7c71 1430 param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
d3c9bc27
TT		 1431}
1432
1433static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
1434 struct mlx5e_sq_param *param)
1435{
1436 void *sqc = param->sqc;
1437 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1438
1439 mlx5e_build_sq_param_common(priv, param);
1440 MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
1441
58d52291 1442 param->max_inline = priv->params.tx_max_inline;
cff92d7c 1443 param->min_inline_mode = priv->params.tx_min_inline_mode;
f62b8bb8
AV		 1444}
1445
1446static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1447 struct mlx5e_cq_param *param)
1448{
1449 void *cqc = param->cqc;
1450
b50d292b 1451 MLX5_SET(cqc, cqc, uar_page, priv->mdev->mlx5e_res.cq_uar.index);
f62b8bb8
AV		 1452}
1453
1454static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
1455 struct mlx5e_cq_param *param)
1456{
1457 void *cqc = param->cqc;
461017cb 1458 u8 log_cq_size;
f62b8bb8 1459
461017cb
TT		 1460 switch (priv->params.rq_wq_type) {
1461 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1462 log_cq_size = priv->params.log_rq_size +
d9d9f156 1463 priv->params.mpwqe_log_num_strides;
461017cb
TT		 1464 break;
1465 default: /* MLX5_WQ_TYPE_LINKED_LIST */
1466 log_cq_size = priv->params.log_rq_size;
1467 }
1468
1469 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
7219ab34
TT		 1470 if (priv->params.rx_cqe_compress) {
1471 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
1472 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
1473 }
f62b8bb8
AV		 1474
1475 mlx5e_build_common_cq_param(priv, param);
9908aa29
TT		 1476
1477 param->cq_period_mode = priv->params.rx_cq_period_mode;
f62b8bb8
AV		 1478}
1479
1480static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
1481 struct mlx5e_cq_param *param)
1482{
1483 void *cqc = param->cqc;
1484
d3c9bc27 1485 MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
f62b8bb8
AV		 1486
1487 mlx5e_build_common_cq_param(priv, param);
9908aa29
TT		 1488
1489 param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
f62b8bb8
AV		 1490}
1491
d3c9bc27
TT		 1492static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
1493 struct mlx5e_cq_param *param,
1494 u8 log_wq_size)
1495{
1496 void *cqc = param->cqc;
1497
1498 MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
1499
1500 mlx5e_build_common_cq_param(priv, param);
9908aa29
TT		 1501
1502 param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
d3c9bc27
TT		 1503}
1504
1505static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
1506 struct mlx5e_sq_param *param,
1507 u8 log_wq_size)
1508{
1509 void *sqc = param->sqc;
1510 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1511
1512 mlx5e_build_sq_param_common(priv, param);
1513
1514 MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
bc77b240 1515 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
d3c9bc27
TT		 1516
1517 param->icosq = true;
1518}
1519
6b87663f 1520static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam)
f62b8bb8 1521{
bc77b240 1522 u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
d3c9bc27 1523
f62b8bb8
AV		 1524 mlx5e_build_rq_param(priv, &cparam->rq);
1525 mlx5e_build_sq_param(priv, &cparam->sq);
d3c9bc27 1526 mlx5e_build_icosq_param(priv, &cparam->icosq, icosq_log_wq_sz);
f62b8bb8
AV		 1527 mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
1528 mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
d3c9bc27 1529 mlx5e_build_ico_cq_param(priv, &cparam->icosq_cq, icosq_log_wq_sz);
f62b8bb8
AV		 1530}
1531
1532static int mlx5e_open_channels(struct mlx5e_priv *priv)
1533{
6b87663f 1534 struct mlx5e_channel_param *cparam;
a4418a6c 1535 int nch = priv->params.num_channels;
03289b88 1536 int err = -ENOMEM;
f62b8bb8
AV		 1537 int i;
1538 int j;
1539
a4418a6c
AS		 1540 priv->channel = kcalloc(nch, sizeof(struct mlx5e_channel *),
1541 GFP_KERNEL);
03289b88 1542
a4418a6c 1543 priv->txq_to_sq_map = kcalloc(nch * priv->params.num_tc,
03289b88
SM		 1544 sizeof(struct mlx5e_sq *), GFP_KERNEL);
1545
6b87663f
AB		 1546 cparam = kzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
1547
1548 if (!priv->channel || !priv->txq_to_sq_map || !cparam)
03289b88 1549 goto err_free_txq_to_sq_map;
f62b8bb8 1550
6b87663f
AB		 1551 mlx5e_build_channel_param(priv, cparam);
1552
a4418a6c 1553 for (i = 0; i < nch; i++) {
6b87663f 1554 err = mlx5e_open_channel(priv, i, cparam, &priv->channel[i]);
f62b8bb8
AV		 1555 if (err)
1556 goto err_close_channels;
1557 }
1558
a4418a6c 1559 for (j = 0; j < nch; j++) {
f62b8bb8
AV		 1560 err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
1561 if (err)
1562 goto err_close_channels;
1563 }
1564
c3b7c5c9
MHY		 1565 /* FIXME: This is a W/A for tx timeout watch dog false alarm when
1566 * polling for inactive tx queues.
1567 */
1568 netif_tx_start_all_queues(priv->netdev);
1569
6b87663f 1570 kfree(cparam);
f62b8bb8
AV		 1571 return 0;
1572
1573err_close_channels:
1574 for (i--; i >= 0; i--)
1575 mlx5e_close_channel(priv->channel[i]);
1576
03289b88
SM		 1577err_free_txq_to_sq_map:
1578 kfree(priv->txq_to_sq_map);
f62b8bb8 1579 kfree(priv->channel);
6b87663f 1580 kfree(cparam);
f62b8bb8
AV		 1581
1582 return err;
1583}
1584
1585static void mlx5e_close_channels(struct mlx5e_priv *priv)
1586{
1587 int i;
1588
c3b7c5c9
MHY		 1589 /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
1590 * polling for inactive tx queues.
1591 */
1592 netif_tx_stop_all_queues(priv->netdev);
1593 netif_tx_disable(priv->netdev);
1594
f62b8bb8
AV		 1595 for (i = 0; i < priv->params.num_channels; i++)
1596 mlx5e_close_channel(priv->channel[i]);
1597
03289b88 1598 kfree(priv->txq_to_sq_map);
f62b8bb8
AV		 1599 kfree(priv->channel);
1600}
1601
2be6967c
SM		 1602static int mlx5e_rx_hash_fn(int hfunc)
1603{
1604 return (hfunc == ETH_RSS_HASH_TOP) ?
1605 MLX5_RX_HASH_FN_TOEPLITZ :
1606 MLX5_RX_HASH_FN_INVERTED_XOR8;
1607}
1608
1609static int mlx5e_bits_invert(unsigned long a, int size)
1610{
1611 int inv = 0;
1612 int i;
1613
1614 for (i = 0; i < size; i++)
1615 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
1616
1617 return inv;
1618}
1619
936896e9
AS		 1620static void mlx5e_fill_indir_rqt_rqns(struct mlx5e_priv *priv, void *rqtc)
1621{
1622 int i;
1623
1624 for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) {
1625 int ix = i;
1da36696 1626 u32 rqn;
936896e9
AS		 1627
1628 if (priv->params.rss_hfunc == ETH_RSS_HASH_XOR)
1629 ix = mlx5e_bits_invert(i, MLX5E_LOG_INDIR_RQT_SIZE);
1630
2d75b2bc 1631 ix = priv->params.indirection_rqt[ix];
1da36696
TT		 1632 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1633 priv->channel[ix]->rq.rqn :
1634 priv->drop_rq.rqn;
1635 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
936896e9
AS		 1636 }
1637}
1638
1da36696
TT		 1639static void mlx5e_fill_direct_rqt_rqn(struct mlx5e_priv *priv, void *rqtc,
1640 int ix)
4cbeaff5 1641{
1da36696
TT		 1642 u32 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1643 priv->channel[ix]->rq.rqn :
1644 priv->drop_rq.rqn;
4cbeaff5 1645
1da36696 1646 MLX5_SET(rqtc, rqtc, rq_num[0], rqn);
4cbeaff5
AS		 1647}
1648
398f3351
HHZ		 1649static int mlx5e_create_rqt(struct mlx5e_priv *priv, int sz,
1650 int ix, struct mlx5e_rqt *rqt)
f62b8bb8
AV		 1651{
1652 struct mlx5_core_dev *mdev = priv->mdev;
f62b8bb8
AV		 1653 void *rqtc;
1654 int inlen;
1655 int err;
1da36696 1656 u32 *in;
f62b8bb8 1657
f62b8bb8
AV		 1658 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1659 in = mlx5_vzalloc(inlen);
1660 if (!in)
1661 return -ENOMEM;
1662
1663 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1664
1665 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1666 MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
1667
1da36696
TT		 1668 if (sz > 1) /* RSS */
1669 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1670 else
1671 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
2be6967c 1672
398f3351
HHZ		 1673 err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
1674 if (!err)
1675 rqt->enabled = true;
f62b8bb8
AV		 1676
1677 kvfree(in);
1da36696
TT		 1678 return err;
1679}
1680
cb67b832 1681void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
1da36696 1682{
398f3351
HHZ		 1683 rqt->enabled = false;
1684 mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
1da36696
TT		 1685}
1686
6bfd390b
HHZ		 1687static int mlx5e_create_indirect_rqts(struct mlx5e_priv *priv)
1688{
1689 struct mlx5e_rqt *rqt = &priv->indir_rqt;
1690
1691 return mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, 0, rqt);
1692}
1693
cb67b832 1694int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
1da36696 1695{
398f3351 1696 struct mlx5e_rqt *rqt;
1da36696
TT		 1697 int err;
1698 int ix;
1699
6bfd390b 1700 for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
398f3351
HHZ		 1701 rqt = &priv->direct_tir[ix].rqt;
1702 err = mlx5e_create_rqt(priv, 1 /*size */, ix, rqt);
1da36696
TT		 1703 if (err)
1704 goto err_destroy_rqts;
1705 }
1706
1707 return 0;
1708
1709err_destroy_rqts:
1710 for (ix--; ix >= 0; ix--)
398f3351 1711 mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt);
1da36696 1712
f62b8bb8
AV		 1713 return err;
1714}
1715
1da36696 1716int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix)
5c50368f
AS		 1717{
1718 struct mlx5_core_dev *mdev = priv->mdev;
5c50368f
AS		 1719 void *rqtc;
1720 int inlen;
1da36696 1721 u32 *in;
5c50368f
AS		 1722 int err;
1723
5c50368f
AS		 1724 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
1725 in = mlx5_vzalloc(inlen);
1726 if (!in)
1727 return -ENOMEM;
1728
1729 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
1730
1731 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1da36696
TT		 1732 if (sz > 1) /* RSS */
1733 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1734 else
1735 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
5c50368f
AS		 1736
1737 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
1738
1da36696 1739 err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
5c50368f
AS		 1740
1741 kvfree(in);
1742
1743 return err;
1744}
1745
40ab6a6e
AS		 1746static void mlx5e_redirect_rqts(struct mlx5e_priv *priv)
1747{
1da36696
TT		 1748 u32 rqtn;
1749 int ix;
1750
398f3351
HHZ		 1751 if (priv->indir_rqt.enabled) {
1752 rqtn = priv->indir_rqt.rqtn;
1753 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0);
1754 }
1755
1da36696 1756 for (ix = 0; ix < priv->params.num_channels; ix++) {
398f3351
HHZ		 1757 if (!priv->direct_tir[ix].rqt.enabled)
1758 continue;
1759 rqtn = priv->direct_tir[ix].rqt.rqtn;
1da36696
TT		 1760 mlx5e_redirect_rqt(priv, rqtn, 1, ix);
1761 }
40ab6a6e
AS		 1762}
1763
5c50368f
AS		 1764static void mlx5e_build_tir_ctx_lro(void *tirc, struct mlx5e_priv *priv)
1765{
1766 if (!priv->params.lro_en)
1767 return;
1768
1769#define ROUGH_MAX_L2_L3_HDR_SZ 256
1770
1771 MLX5_SET(tirc, tirc, lro_enable_mask,
1772 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
1773 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
1774 MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
1775 (priv->params.lro_wqe_sz -
1776 ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
1777 MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
1778 MLX5_CAP_ETH(priv->mdev,
d9a40271 1779 lro_timer_supported_periods[2]));
5c50368f
AS		 1780}
1781
bdfc028d
TT		 1782void mlx5e_build_tir_ctx_hash(void *tirc, struct mlx5e_priv *priv)
1783{
1784 MLX5_SET(tirc, tirc, rx_hash_fn,
1785 mlx5e_rx_hash_fn(priv->params.rss_hfunc));
1786 if (priv->params.rss_hfunc == ETH_RSS_HASH_TOP) {
1787 void *rss_key = MLX5_ADDR_OF(tirc, tirc,
1788 rx_hash_toeplitz_key);
1789 size_t len = MLX5_FLD_SZ_BYTES(tirc,
1790 rx_hash_toeplitz_key);
1791
1792 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1793 memcpy(rss_key, priv->params.toeplitz_hash_key, len);
1794 }
1795}
1796
ab0394fe 1797static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
5c50368f
AS		 1798{
1799 struct mlx5_core_dev *mdev = priv->mdev;
1800
1801 void *in;
1802 void *tirc;
1803 int inlen;
1804 int err;
ab0394fe 1805 int tt;
1da36696 1806 int ix;
5c50368f
AS		 1807
1808 inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
1809 in = mlx5_vzalloc(inlen);
1810 if (!in)
1811 return -ENOMEM;
1812
1813 MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
1814 tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
1815
1816 mlx5e_build_tir_ctx_lro(tirc, priv);
1817
1da36696 1818 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
724b2aa1 1819 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in,
1da36696 1820 inlen);
ab0394fe 1821 if (err)
1da36696 1822 goto free_in;
ab0394fe 1823 }
5c50368f 1824
6bfd390b 1825 for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
1da36696
TT		 1826 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn,
1827 in, inlen);
1828 if (err)
1829 goto free_in;
1830 }
1831
1832free_in:
5c50368f
AS		 1833 kvfree(in);
1834
1835 return err;
1836}
1837
cd255eff 1838static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu)
40ab6a6e 1839{
40ab6a6e 1840 struct mlx5_core_dev *mdev = priv->mdev;
cd255eff 1841 u16 hw_mtu = MLX5E_SW2HW_MTU(mtu);
40ab6a6e
AS		 1842 int err;
1843
cd255eff 1844 err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
40ab6a6e
AS		 1845 if (err)
1846 return err;
1847
cd255eff
SM		 1848 /* Update vport context MTU */
1849 mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
1850 return 0;
1851}
40ab6a6e 1852
cd255eff
SM		 1853static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu)
1854{
1855 struct mlx5_core_dev *mdev = priv->mdev;
1856 u16 hw_mtu = 0;
1857 int err;
40ab6a6e 1858
cd255eff
SM		 1859 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
1860 if (err || !hw_mtu) /* fallback to port oper mtu */
1861 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
1862
1863 *mtu = MLX5E_HW2SW_MTU(hw_mtu);
1864}
1865
1866static int mlx5e_set_dev_port_mtu(struct net_device *netdev)
1867{
1868 struct mlx5e_priv *priv = netdev_priv(netdev);
1869 u16 mtu;
1870 int err;
1871
1872 err = mlx5e_set_mtu(priv, netdev->mtu);
1873 if (err)
1874 return err;
40ab6a6e 1875
cd255eff
SM		 1876 mlx5e_query_mtu(priv, &mtu);
1877 if (mtu != netdev->mtu)
1878 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
1879 __func__, mtu, netdev->mtu);
40ab6a6e 1880
cd255eff 1881 netdev->mtu = mtu;
40ab6a6e
AS		 1882 return 0;
1883}
1884
08fb1dac
SM		 1885static void mlx5e_netdev_set_tcs(struct net_device *netdev)
1886{
1887 struct mlx5e_priv *priv = netdev_priv(netdev);
1888 int nch = priv->params.num_channels;
1889 int ntc = priv->params.num_tc;
1890 int tc;
1891
1892 netdev_reset_tc(netdev);
1893
1894 if (ntc == 1)
1895 return;
1896
1897 netdev_set_num_tc(netdev, ntc);
1898
7ccdd084
RS		 1899 /* Map netdev TCs to offset 0
1900 * We have our own UP to TXQ mapping for QoS
1901 */
08fb1dac 1902 for (tc = 0; tc < ntc; tc++)
7ccdd084 1903 netdev_set_tc_queue(netdev, tc, nch, 0);
08fb1dac
SM		 1904}
1905
40ab6a6e
AS		 1906int mlx5e_open_locked(struct net_device *netdev)
1907{
1908 struct mlx5e_priv *priv = netdev_priv(netdev);
cb67b832 1909 struct mlx5_core_dev *mdev = priv->mdev;
40ab6a6e
AS		 1910 int num_txqs;
1911 int err;
1912
1913 set_bit(MLX5E_STATE_OPENED, &priv->state);
1914
08fb1dac
SM		 1915 mlx5e_netdev_set_tcs(netdev);
1916
40ab6a6e
AS		 1917 num_txqs = priv->params.num_channels * priv->params.num_tc;
1918 netif_set_real_num_tx_queues(netdev, num_txqs);
1919 netif_set_real_num_rx_queues(netdev, priv->params.num_channels);
1920
40ab6a6e
AS		 1921 err = mlx5e_open_channels(priv);
1922 if (err) {
1923 netdev_err(netdev, "%s: mlx5e_open_channels failed, %d\n",
1924 __func__, err);
343b29f3 1925 goto err_clear_state_opened_flag;
40ab6a6e
AS		 1926 }
1927
724b2aa1 1928 err = mlx5e_refresh_tirs_self_loopback_enable(priv->mdev);
66189961
TT		 1929 if (err) {
1930 netdev_err(netdev, "%s: mlx5e_refresh_tirs_self_loopback_enable failed, %d\n",
1931 __func__, err);
1932 goto err_close_channels;
1933 }
1934
40ab6a6e 1935 mlx5e_redirect_rqts(priv);
ce89ef36 1936 mlx5e_update_carrier(priv);
ef9814de 1937 mlx5e_timestamp_init(priv);
5a7b27eb
MG		 1938#ifdef CONFIG_RFS_ACCEL
1939 priv->netdev->rx_cpu_rmap = priv->mdev->rmap;
1940#endif
cb67b832
HHZ		 1941 if (priv->profile->update_stats)
1942 queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
40ab6a6e 1943
cb67b832
HHZ		 1944 if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
1945 err = mlx5e_add_sqs_fwd_rules(priv);
1946 if (err)
1947 goto err_close_channels;
1948 }
9b37b07f 1949 return 0;
343b29f3 1950
66189961
TT		 1951err_close_channels:
1952 mlx5e_close_channels(priv);
343b29f3
AS		 1953err_clear_state_opened_flag:
1954 clear_bit(MLX5E_STATE_OPENED, &priv->state);
1955 return err;
40ab6a6e
AS		 1956}
1957
cb67b832 1958int mlx5e_open(struct net_device *netdev)
40ab6a6e
AS		 1959{
1960 struct mlx5e_priv *priv = netdev_priv(netdev);
1961 int err;
1962
1963 mutex_lock(&priv->state_lock);
1964 err = mlx5e_open_locked(netdev);
1965 mutex_unlock(&priv->state_lock);
1966
1967 return err;
1968}
1969
1970int mlx5e_close_locked(struct net_device *netdev)
1971{
1972 struct mlx5e_priv *priv = netdev_priv(netdev);
cb67b832 1973 struct mlx5_core_dev *mdev = priv->mdev;
40ab6a6e 1974
a1985740
AS		 1975 /* May already be CLOSED in case a previous configuration operation
1976 * (e.g RX/TX queue size change) that involves close&open failed.
1977 */
1978 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
1979 return 0;
1980
40ab6a6e
AS		 1981 clear_bit(MLX5E_STATE_OPENED, &priv->state);
1982
cb67b832
HHZ		 1983 if (MLX5_CAP_GEN(mdev, vport_group_manager))
1984 mlx5e_remove_sqs_fwd_rules(priv);
1985
ef9814de 1986 mlx5e_timestamp_cleanup(priv);
40ab6a6e 1987 netif_carrier_off(priv->netdev);
ce89ef36 1988 mlx5e_redirect_rqts(priv);
40ab6a6e
AS		 1989 mlx5e_close_channels(priv);
1990
1991 return 0;
1992}
1993
cb67b832 1994int mlx5e_close(struct net_device *netdev)
40ab6a6e
AS		 1995{
1996 struct mlx5e_priv *priv = netdev_priv(netdev);
1997 int err;
1998
26e59d80
MHY		 1999 if (!netif_device_present(netdev))
2000 return -ENODEV;
2001
40ab6a6e
AS		 2002 mutex_lock(&priv->state_lock);
2003 err = mlx5e_close_locked(netdev);
2004 mutex_unlock(&priv->state_lock);
2005
2006 return err;
2007}
2008
2009static int mlx5e_create_drop_rq(struct mlx5e_priv *priv,
2010 struct mlx5e_rq *rq,
2011 struct mlx5e_rq_param *param)
2012{
2013 struct mlx5_core_dev *mdev = priv->mdev;
2014 void *rqc = param->rqc;
2015 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
2016 int err;
2017
2018 param->wq.db_numa_node = param->wq.buf_numa_node;
2019
2020 err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
2021 &rq->wq_ctrl);
2022 if (err)
2023 return err;
2024
2025 rq->priv = priv;
2026
2027 return 0;
2028}
2029
2030static int mlx5e_create_drop_cq(struct mlx5e_priv *priv,
2031 struct mlx5e_cq *cq,
2032 struct mlx5e_cq_param *param)
2033{
2034 struct mlx5_core_dev *mdev = priv->mdev;
2035 struct mlx5_core_cq *mcq = &cq->mcq;
2036 int eqn_not_used;
0b6e26ce 2037 unsigned int irqn;
40ab6a6e
AS		 2038 int err;
2039
2040 err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
2041 &cq->wq_ctrl);
2042 if (err)
2043 return err;
2044
2045 mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
2046
2047 mcq->cqe_sz = 64;
2048 mcq->set_ci_db = cq->wq_ctrl.db.db;
2049 mcq->arm_db = cq->wq_ctrl.db.db + 1;
2050 *mcq->set_ci_db = 0;
2051 *mcq->arm_db = 0;
2052 mcq->vector = param->eq_ix;
2053 mcq->comp = mlx5e_completion_event;
2054 mcq->event = mlx5e_cq_error_event;
2055 mcq->irqn = irqn;
b50d292b 2056 mcq->uar = &mdev->mlx5e_res.cq_uar;
40ab6a6e
AS		 2057
2058 cq->priv = priv;
2059
2060 return 0;
2061}
2062
2063static int mlx5e_open_drop_rq(struct mlx5e_priv *priv)
2064{
2065 struct mlx5e_cq_param cq_param;
2066 struct mlx5e_rq_param rq_param;
2067 struct mlx5e_rq *rq = &priv->drop_rq;
2068 struct mlx5e_cq *cq = &priv->drop_rq.cq;
2069 int err;
2070
2071 memset(&cq_param, 0, sizeof(cq_param));
2072 memset(&rq_param, 0, sizeof(rq_param));
556dd1b9 2073 mlx5e_build_drop_rq_param(&rq_param);
40ab6a6e
AS		 2074
2075 err = mlx5e_create_drop_cq(priv, cq, &cq_param);
2076 if (err)
2077 return err;
2078
2079 err = mlx5e_enable_cq(cq, &cq_param);
2080 if (err)
2081 goto err_destroy_cq;
2082
2083 err = mlx5e_create_drop_rq(priv, rq, &rq_param);
2084 if (err)
2085 goto err_disable_cq;
2086
2087 err = mlx5e_enable_rq(rq, &rq_param);
2088 if (err)
2089 goto err_destroy_rq;
2090
2091 return 0;
2092
2093err_destroy_rq:
2094 mlx5e_destroy_rq(&priv->drop_rq);
2095
2096err_disable_cq:
2097 mlx5e_disable_cq(&priv->drop_rq.cq);
2098
2099err_destroy_cq:
2100 mlx5e_destroy_cq(&priv->drop_rq.cq);
2101
2102 return err;
2103}
2104
2105static void mlx5e_close_drop_rq(struct mlx5e_priv *priv)
2106{
2107 mlx5e_disable_rq(&priv->drop_rq);
2108 mlx5e_destroy_rq(&priv->drop_rq);
2109 mlx5e_disable_cq(&priv->drop_rq.cq);
2110 mlx5e_destroy_cq(&priv->drop_rq.cq);
2111}
2112
2113static int mlx5e_create_tis(struct mlx5e_priv *priv, int tc)
2114{
2115 struct mlx5_core_dev *mdev = priv->mdev;
c4f287c4 2116 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
40ab6a6e
AS		 2117 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2118
08fb1dac 2119 MLX5_SET(tisc, tisc, prio, tc << 1);
b50d292b 2120 MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
db60b802
AH		 2121
2122 if (mlx5_lag_is_lacp_owner(mdev))
2123 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
2124
40ab6a6e
AS		 2125 return mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]);
2126}
2127
2128static void mlx5e_destroy_tis(struct mlx5e_priv *priv, int tc)
2129{
2130 mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
2131}
2132
cb67b832 2133int mlx5e_create_tises(struct mlx5e_priv *priv)
40ab6a6e
AS		 2134{
2135 int err;
2136 int tc;
2137
6bfd390b 2138 for (tc = 0; tc < priv->profile->max_tc; tc++) {
40ab6a6e
AS		 2139 err = mlx5e_create_tis(priv, tc);
2140 if (err)
2141 goto err_close_tises;
2142 }
2143
2144 return 0;
2145
2146err_close_tises:
2147 for (tc--; tc >= 0; tc--)
2148 mlx5e_destroy_tis(priv, tc);
2149
2150 return err;
2151}
2152
cb67b832 2153void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
40ab6a6e
AS		 2154{
2155 int tc;
2156
6bfd390b 2157 for (tc = 0; tc < priv->profile->max_tc; tc++)
40ab6a6e
AS		 2158 mlx5e_destroy_tis(priv, tc);
2159}
2160
1da36696
TT		 2161static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2162 enum mlx5e_traffic_types tt)
f62b8bb8
AV		 2163{
2164 void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2165
b50d292b 2166 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
3191e05f 2167
5a6f8aef
AS		 2168#define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\
2169 MLX5_HASH_FIELD_SEL_DST_IP)
f62b8bb8 2170
5a6f8aef
AS		 2171#define MLX5_HASH_IP_L4PORTS (MLX5_HASH_FIELD_SEL_SRC_IP |\
2172 MLX5_HASH_FIELD_SEL_DST_IP |\
2173 MLX5_HASH_FIELD_SEL_L4_SPORT |\
2174 MLX5_HASH_FIELD_SEL_L4_DPORT)
f62b8bb8 2175
a741749f
AS		 2176#define MLX5_HASH_IP_IPSEC_SPI (MLX5_HASH_FIELD_SEL_SRC_IP |\
2177 MLX5_HASH_FIELD_SEL_DST_IP |\
2178 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2179
5c50368f 2180 mlx5e_build_tir_ctx_lro(tirc, priv);
f62b8bb8 2181
4cbeaff5 2182 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
398f3351 2183 MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
1da36696 2184 mlx5e_build_tir_ctx_hash(tirc, priv);
f62b8bb8
AV		 2185
2186 switch (tt) {
2187 case MLX5E_TT_IPV4_TCP:
2188 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2189 MLX5_L3_PROT_TYPE_IPV4);
2190 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2191 MLX5_L4_PROT_TYPE_TCP);
2192 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
5a6f8aef 2193 MLX5_HASH_IP_L4PORTS);
f62b8bb8
AV		 2194 break;
2195
2196 case MLX5E_TT_IPV6_TCP:
2197 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2198 MLX5_L3_PROT_TYPE_IPV6);
2199 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2200 MLX5_L4_PROT_TYPE_TCP);
2201 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
5a6f8aef 2202 MLX5_HASH_IP_L4PORTS);
f62b8bb8
AV		 2203 break;
2204
2205 case MLX5E_TT_IPV4_UDP:
2206 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2207 MLX5_L3_PROT_TYPE_IPV4);
2208 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2209 MLX5_L4_PROT_TYPE_UDP);
2210 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
5a6f8aef 2211 MLX5_HASH_IP_L4PORTS);
f62b8bb8
AV		 2212 break;
2213
2214 case MLX5E_TT_IPV6_UDP:
2215 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2216 MLX5_L3_PROT_TYPE_IPV6);
2217 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2218 MLX5_L4_PROT_TYPE_UDP);
2219 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
5a6f8aef 2220 MLX5_HASH_IP_L4PORTS);
f62b8bb8
AV		 2221 break;
2222
a741749f
AS		 2223 case MLX5E_TT_IPV4_IPSEC_AH:
2224 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2225 MLX5_L3_PROT_TYPE_IPV4);
2226 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2227 MLX5_HASH_IP_IPSEC_SPI);
2228 break;
2229
2230 case MLX5E_TT_IPV6_IPSEC_AH:
2231 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2232 MLX5_L3_PROT_TYPE_IPV6);
2233 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2234 MLX5_HASH_IP_IPSEC_SPI);
2235 break;
2236
2237 case MLX5E_TT_IPV4_IPSEC_ESP:
2238 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2239 MLX5_L3_PROT_TYPE_IPV4);
2240 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2241 MLX5_HASH_IP_IPSEC_SPI);
2242 break;
2243
2244 case MLX5E_TT_IPV6_IPSEC_ESP:
2245 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2246 MLX5_L3_PROT_TYPE_IPV6);
2247 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2248 MLX5_HASH_IP_IPSEC_SPI);
2249 break;
2250
f62b8bb8
AV		 2251 case MLX5E_TT_IPV4:
2252 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2253 MLX5_L3_PROT_TYPE_IPV4);
2254 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2255 MLX5_HASH_IP);
2256 break;
2257
2258 case MLX5E_TT_IPV6:
2259 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2260 MLX5_L3_PROT_TYPE_IPV6);
2261 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2262 MLX5_HASH_IP);
2263 break;
1da36696
TT		 2264 default:
2265 WARN_ONCE(true,
2266 "mlx5e_build_indir_tir_ctx: bad traffic type!\n");
f62b8bb8
AV		 2267 }
2268}
2269
1da36696
TT		 2270static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2271 u32 rqtn)
f62b8bb8 2272{
b50d292b 2273 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
1da36696
TT		 2274
2275 mlx5e_build_tir_ctx_lro(tirc, priv);
2276
2277 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2278 MLX5_SET(tirc, tirc, indirect_table, rqtn);
2279 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
2280}
2281
6bfd390b 2282static int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
1da36696 2283{
724b2aa1 2284 struct mlx5e_tir *tir;
f62b8bb8
AV		 2285 void *tirc;
2286 int inlen;
2287 int err;
1da36696 2288 u32 *in;
1da36696 2289 int tt;
f62b8bb8
AV		 2290
2291 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2292 in = mlx5_vzalloc(inlen);
2293 if (!in)
2294 return -ENOMEM;
2295
1da36696
TT		 2296 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2297 memset(in, 0, inlen);
724b2aa1 2298 tir = &priv->indir_tir[tt];
1da36696
TT		 2299 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2300 mlx5e_build_indir_tir_ctx(priv, tirc, tt);
724b2aa1 2301 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
f62b8bb8 2302 if (err)
40ab6a6e 2303 goto err_destroy_tirs;
f62b8bb8
AV		 2304 }
2305
6bfd390b
HHZ		 2306 kvfree(in);
2307
2308 return 0;
2309
2310err_destroy_tirs:
2311 for (tt--; tt >= 0; tt--)
2312 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
2313
2314 kvfree(in);
2315
2316 return err;
2317}
2318
cb67b832 2319int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)
6bfd390b
HHZ		 2320{
2321 int nch = priv->profile->max_nch(priv->mdev);
2322 struct mlx5e_tir *tir;
2323 void *tirc;
2324 int inlen;
2325 int err;
2326 u32 *in;
2327 int ix;
2328
2329 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2330 in = mlx5_vzalloc(inlen);
2331 if (!in)
2332 return -ENOMEM;
2333
1da36696
TT		 2334 for (ix = 0; ix < nch; ix++) {
2335 memset(in, 0, inlen);
724b2aa1 2336 tir = &priv->direct_tir[ix];
1da36696
TT		 2337 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2338 mlx5e_build_direct_tir_ctx(priv, tirc,
398f3351 2339 priv->direct_tir[ix].rqt.rqtn);
724b2aa1 2340 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
1da36696
TT		 2341 if (err)
2342 goto err_destroy_ch_tirs;
2343 }
2344
2345 kvfree(in);
2346
f62b8bb8
AV		 2347 return 0;
2348
1da36696
TT		 2349err_destroy_ch_tirs:
2350 for (ix--; ix >= 0; ix--)
724b2aa1 2351 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]);
1da36696 2352
1da36696 2353 kvfree(in);
f62b8bb8
AV		 2354
2355 return err;
2356}
2357
6bfd390b 2358static void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
f62b8bb8
AV		 2359{
2360 int i;
2361
1da36696 2362 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
724b2aa1 2363 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
f62b8bb8
AV		 2364}
2365
cb67b832 2366void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
6bfd390b
HHZ		 2367{
2368 int nch = priv->profile->max_nch(priv->mdev);
2369 int i;
2370
2371 for (i = 0; i < nch; i++)
2372 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);
2373}
2374
36350114
GP		 2375int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd)
2376{
2377 int err = 0;
2378 int i;
2379
2380 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2381 return 0;
2382
2383 for (i = 0; i < priv->params.num_channels; i++) {
2384 err = mlx5e_modify_rq_vsd(&priv->channel[i]->rq, vsd);
2385 if (err)
2386 return err;
2387 }
2388
2389 return 0;
2390}
2391
08fb1dac
SM		 2392static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
2393{
2394 struct mlx5e_priv *priv = netdev_priv(netdev);
2395 bool was_opened;
2396 int err = 0;
2397
2398 if (tc && tc != MLX5E_MAX_NUM_TC)
2399 return -EINVAL;
2400
2401 mutex_lock(&priv->state_lock);
2402
2403 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2404 if (was_opened)
2405 mlx5e_close_locked(priv->netdev);
2406
2407 priv->params.num_tc = tc ? tc : 1;
2408
2409 if (was_opened)
2410 err = mlx5e_open_locked(priv->netdev);
2411
2412 mutex_unlock(&priv->state_lock);
2413
2414 return err;
2415}
2416
2417static int mlx5e_ndo_setup_tc(struct net_device *dev, u32 handle,
2418 __be16 proto, struct tc_to_netdev *tc)
2419{
e8f887ac
AV		 2420 struct mlx5e_priv *priv = netdev_priv(dev);
2421
2422 if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
2423 goto mqprio;
2424
2425 switch (tc->type) {
e3a2b7ed
AV		 2426 case TC_SETUP_CLSFLOWER:
2427 switch (tc->cls_flower->command) {
2428 case TC_CLSFLOWER_REPLACE:
2429 return mlx5e_configure_flower(priv, proto, tc->cls_flower);
2430 case TC_CLSFLOWER_DESTROY:
2431 return mlx5e_delete_flower(priv, tc->cls_flower);
aad7e08d
AV		 2432 case TC_CLSFLOWER_STATS:
2433 return mlx5e_stats_flower(priv, tc->cls_flower);
e3a2b7ed 2434 }
e8f887ac
AV		 2435 default:
2436 return -EOPNOTSUPP;
2437 }
2438
2439mqprio:
67ba422e 2440 if (tc->type != TC_SETUP_MQPRIO)
08fb1dac
SM		 2441 return -EINVAL;
2442
2443 return mlx5e_setup_tc(dev, tc->tc);
2444}
2445
cb67b832 2446struct rtnl_link_stats64 *
f62b8bb8
AV		 2447mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
2448{
2449 struct mlx5e_priv *priv = netdev_priv(dev);
9218b44d 2450 struct mlx5e_sw_stats *sstats = &priv->stats.sw;
f62b8bb8 2451 struct mlx5e_vport_stats *vstats = &priv->stats.vport;
269e6b3a 2452 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
f62b8bb8 2453
9218b44d
GP		 2454 stats->rx_packets = sstats->rx_packets;
2455 stats->rx_bytes = sstats->rx_bytes;
2456 stats->tx_packets = sstats->tx_packets;
2457 stats->tx_bytes = sstats->tx_bytes;
269e6b3a
GP		 2458
2459 stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
9218b44d 2460 stats->tx_dropped = sstats->tx_queue_dropped;
269e6b3a
GP		 2461
2462 stats->rx_length_errors =
9218b44d
GP		 2463 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
2464 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
2465 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
269e6b3a 2466 stats->rx_crc_errors =
9218b44d
GP		 2467 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
2468 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
2469 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
269e6b3a 2470 stats->tx_carrier_errors =
9218b44d 2471 PPORT_802_3_GET(pstats, a_symbol_error_during_carrier);
269e6b3a
GP		 2472 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
2473 stats->rx_frame_errors;
2474 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
2475
2476 /* vport multicast also counts packets that are dropped due to steering
2477 * or rx out of buffer
2478 */
9218b44d
GP		 2479 stats->multicast =
2480 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
f62b8bb8
AV		 2481
2482 return stats;
2483}
2484
2485static void mlx5e_set_rx_mode(struct net_device *dev)
2486{
2487 struct mlx5e_priv *priv = netdev_priv(dev);
2488
7bb29755 2489 queue_work(priv->wq, &priv->set_rx_mode_work);
f62b8bb8
AV		 2490}
2491
2492static int mlx5e_set_mac(struct net_device *netdev, void *addr)
2493{
2494 struct mlx5e_priv *priv = netdev_priv(netdev);
2495 struct sockaddr *saddr = addr;
2496
2497 if (!is_valid_ether_addr(saddr->sa_data))
2498 return -EADDRNOTAVAIL;
2499
2500 netif_addr_lock_bh(netdev);
2501 ether_addr_copy(netdev->dev_addr, saddr->sa_data);
2502 netif_addr_unlock_bh(netdev);
2503
7bb29755 2504 queue_work(priv->wq, &priv->set_rx_mode_work);
f62b8bb8
AV		 2505
2506 return 0;
2507}
2508
0e405443
GP		 2509#define MLX5E_SET_FEATURE(netdev, feature, enable) \
2510 do { \
2511 if (enable) \
2512 netdev->features |= feature; \
2513 else \
2514 netdev->features &= ~feature; \
2515 } while (0)
2516
2517typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
2518
2519static int set_feature_lro(struct net_device *netdev, bool enable)
f62b8bb8
AV		 2520{
2521 struct mlx5e_priv *priv = netdev_priv(netdev);
0e405443
GP		 2522 bool was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2523 int err;
f62b8bb8
AV		 2524
2525 mutex_lock(&priv->state_lock);
f62b8bb8 2526
0e405443
GP		 2527 if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2528 mlx5e_close_locked(priv->netdev);
98e81b0a 2529
0e405443
GP		 2530 priv->params.lro_en = enable;
2531 err = mlx5e_modify_tirs_lro(priv);
2532 if (err) {
2533 netdev_err(netdev, "lro modify failed, %d\n", err);
2534 priv->params.lro_en = !enable;
98e81b0a 2535 }
f62b8bb8 2536
0e405443
GP		 2537 if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2538 mlx5e_open_locked(priv->netdev);
2539
9b37b07f
AS		 2540 mutex_unlock(&priv->state_lock);
2541
0e405443
GP		 2542 return err;
2543}
2544
2545static int set_feature_vlan_filter(struct net_device *netdev, bool enable)
2546{
2547 struct mlx5e_priv *priv = netdev_priv(netdev);
2548
2549 if (enable)
2550 mlx5e_enable_vlan_filter(priv);
2551 else
2552 mlx5e_disable_vlan_filter(priv);
2553
2554 return 0;
2555}
2556
2557static int set_feature_tc_num_filters(struct net_device *netdev, bool enable)
2558{
2559 struct mlx5e_priv *priv = netdev_priv(netdev);
f62b8bb8 2560
0e405443 2561 if (!enable && mlx5e_tc_num_filters(priv)) {
e8f887ac
AV		 2562 netdev_err(netdev,
2563 "Active offloaded tc filters, can't turn hw_tc_offload off\n");
2564 return -EINVAL;
2565 }
2566
0e405443
GP		 2567 return 0;
2568}
2569
94cb1ebb
EBE		 2570static int set_feature_rx_all(struct net_device *netdev, bool enable)
2571{
2572 struct mlx5e_priv *priv = netdev_priv(netdev);
2573 struct mlx5_core_dev *mdev = priv->mdev;
2574
2575 return mlx5_set_port_fcs(mdev, !enable);
2576}
2577
36350114
GP		 2578static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
2579{
2580 struct mlx5e_priv *priv = netdev_priv(netdev);
2581 int err;
2582
2583 mutex_lock(&priv->state_lock);
2584
2585 priv->params.vlan_strip_disable = !enable;
2586 err = mlx5e_modify_rqs_vsd(priv, !enable);
2587 if (err)
2588 priv->params.vlan_strip_disable = enable;
2589
2590 mutex_unlock(&priv->state_lock);
2591
2592 return err;
2593}
2594
45bf454a
MG		 2595#ifdef CONFIG_RFS_ACCEL
2596static int set_feature_arfs(struct net_device *netdev, bool enable)
2597{
2598 struct mlx5e_priv *priv = netdev_priv(netdev);
2599 int err;
2600
2601 if (enable)
2602 err = mlx5e_arfs_enable(priv);
2603 else
2604 err = mlx5e_arfs_disable(priv);
2605
2606 return err;
2607}
2608#endif
2609
0e405443
GP		 2610static int mlx5e_handle_feature(struct net_device *netdev,
2611 netdev_features_t wanted_features,
2612 netdev_features_t feature,
2613 mlx5e_feature_handler feature_handler)
2614{
2615 netdev_features_t changes = wanted_features ^ netdev->features;
2616 bool enable = !!(wanted_features & feature);
2617 int err;
2618
2619 if (!(changes & feature))
2620 return 0;
2621
2622 err = feature_handler(netdev, enable);
2623 if (err) {
2624 netdev_err(netdev, "%s feature 0x%llx failed err %d\n",
2625 enable ? "Enable" : "Disable", feature, err);
2626 return err;
2627 }
2628
2629 MLX5E_SET_FEATURE(netdev, feature, enable);
2630 return 0;
2631}
2632
2633static int mlx5e_set_features(struct net_device *netdev,
2634 netdev_features_t features)
2635{
2636 int err;
2637
2638 err = mlx5e_handle_feature(netdev, features, NETIF_F_LRO,
2639 set_feature_lro);
2640 err |= mlx5e_handle_feature(netdev, features,
2641 NETIF_F_HW_VLAN_CTAG_FILTER,
2642 set_feature_vlan_filter);
2643 err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_TC,
2644 set_feature_tc_num_filters);
94cb1ebb
EBE		 2645 err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXALL,
2646 set_feature_rx_all);
36350114
GP		 2647 err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_VLAN_CTAG_RX,
2648 set_feature_rx_vlan);
45bf454a
MG		 2649#ifdef CONFIG_RFS_ACCEL
2650 err |= mlx5e_handle_feature(netdev, features, NETIF_F_NTUPLE,
2651 set_feature_arfs);
2652#endif
0e405443
GP		 2653
2654 return err ? -EINVAL : 0;
f62b8bb8
AV		 2655}
2656
d8edd246
SM		 2657#define MXL5_HW_MIN_MTU 64
2658#define MXL5E_MIN_MTU (MXL5_HW_MIN_MTU + ETH_FCS_LEN)
2659
f62b8bb8
AV		 2660static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
2661{
2662 struct mlx5e_priv *priv = netdev_priv(netdev);
2663 struct mlx5_core_dev *mdev = priv->mdev;
98e81b0a 2664 bool was_opened;
046339ea 2665 u16 max_mtu;
d8edd246 2666 u16 min_mtu;
98e81b0a 2667 int err = 0;
506753b0 2668 bool reset;
f62b8bb8 2669
facc9699 2670 mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
f62b8bb8 2671
50a9eea6 2672 max_mtu = MLX5E_HW2SW_MTU(max_mtu);
d8edd246 2673 min_mtu = MLX5E_HW2SW_MTU(MXL5E_MIN_MTU);
50a9eea6 2674
d8edd246 2675 if (new_mtu > max_mtu || new_mtu < min_mtu) {
facc9699 2676 netdev_err(netdev,
d8edd246
SM		 2677 "%s: Bad MTU (%d), valid range is: [%d..%d]\n",
2678 __func__, new_mtu, min_mtu, max_mtu);
f62b8bb8
AV		 2679 return -EINVAL;
2680 }
2681
2682 mutex_lock(&priv->state_lock);
98e81b0a 2683
506753b0
TT		 2684 reset = !priv->params.lro_en &&
2685 (priv->params.rq_wq_type !=
2686 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
2687
98e81b0a 2688 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
506753b0 2689 if (was_opened && reset)
98e81b0a
AS		 2690 mlx5e_close_locked(netdev);
2691
f62b8bb8 2692 netdev->mtu = new_mtu;
13f9bba7 2693 mlx5e_set_dev_port_mtu(netdev);
98e81b0a 2694
506753b0 2695 if (was_opened && reset)
98e81b0a
AS		 2696 err = mlx5e_open_locked(netdev);
2697
f62b8bb8
AV		 2698 mutex_unlock(&priv->state_lock);
2699
2700 return err;
2701}
2702
ef9814de
EBE		 2703static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2704{
2705 switch (cmd) {
2706 case SIOCSHWTSTAMP:
2707 return mlx5e_hwstamp_set(dev, ifr);
2708 case SIOCGHWTSTAMP:
2709 return mlx5e_hwstamp_get(dev, ifr);
2710 default:
2711 return -EOPNOTSUPP;
2712 }
2713}
2714
66e49ded
SM		 2715static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
2716{
2717 struct mlx5e_priv *priv = netdev_priv(dev);
2718 struct mlx5_core_dev *mdev = priv->mdev;
2719
2720 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
2721}
2722
2723static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos)
2724{
2725 struct mlx5e_priv *priv = netdev_priv(dev);
2726 struct mlx5_core_dev *mdev = priv->mdev;
2727
2728 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
2729 vlan, qos);
2730}
2731
f942380c
MHY		 2732static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
2733{
2734 struct mlx5e_priv *priv = netdev_priv(dev);
2735 struct mlx5_core_dev *mdev = priv->mdev;
2736
2737 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
2738}
2739
1edc57e2
MHY		 2740static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
2741{
2742 struct mlx5e_priv *priv = netdev_priv(dev);
2743 struct mlx5_core_dev *mdev = priv->mdev;
2744
2745 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
2746}
66e49ded
SM		 2747static int mlx5_vport_link2ifla(u8 esw_link)
2748{
2749 switch (esw_link) {
2750 case MLX5_ESW_VPORT_ADMIN_STATE_DOWN:
2751 return IFLA_VF_LINK_STATE_DISABLE;
2752 case MLX5_ESW_VPORT_ADMIN_STATE_UP:
2753 return IFLA_VF_LINK_STATE_ENABLE;
2754 }
2755 return IFLA_VF_LINK_STATE_AUTO;
2756}
2757
2758static int mlx5_ifla_link2vport(u8 ifla_link)
2759{
2760 switch (ifla_link) {
2761 case IFLA_VF_LINK_STATE_DISABLE:
2762 return MLX5_ESW_VPORT_ADMIN_STATE_DOWN;
2763 case IFLA_VF_LINK_STATE_ENABLE:
2764 return MLX5_ESW_VPORT_ADMIN_STATE_UP;
2765 }
2766 return MLX5_ESW_VPORT_ADMIN_STATE_AUTO;
2767}
2768
2769static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
2770 int link_state)
2771{
2772 struct mlx5e_priv *priv = netdev_priv(dev);
2773 struct mlx5_core_dev *mdev = priv->mdev;
2774
2775 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
2776 mlx5_ifla_link2vport(link_state));
2777}
2778
2779static int mlx5e_get_vf_config(struct net_device *dev,
2780 int vf, struct ifla_vf_info *ivi)
2781{
2782 struct mlx5e_priv *priv = netdev_priv(dev);
2783 struct mlx5_core_dev *mdev = priv->mdev;
2784 int err;
2785
2786 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
2787 if (err)
2788 return err;
2789 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
2790 return 0;
2791}
2792
2793static int mlx5e_get_vf_stats(struct net_device *dev,
2794 int vf, struct ifla_vf_stats *vf_stats)
2795{
2796 struct mlx5e_priv *priv = netdev_priv(dev);
2797 struct mlx5_core_dev *mdev = priv->mdev;
2798
2799 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
2800 vf_stats);
2801}
2802
b3f63c3d 2803static void mlx5e_add_vxlan_port(struct net_device *netdev,
974c3f30 2804 struct udp_tunnel_info *ti)
b3f63c3d
MF		 2805{
2806 struct mlx5e_priv *priv = netdev_priv(netdev);
2807
974c3f30
AD		 2808 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2809 return;
2810
b3f63c3d
MF		 2811 if (!mlx5e_vxlan_allowed(priv->mdev))
2812 return;
2813
974c3f30 2814 mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 1);
b3f63c3d
MF		 2815}
2816
2817static void mlx5e_del_vxlan_port(struct net_device *netdev,
974c3f30 2818 struct udp_tunnel_info *ti)
b3f63c3d
MF		 2819{
2820 struct mlx5e_priv *priv = netdev_priv(netdev);
2821
974c3f30
AD		 2822 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2823 return;
2824
b3f63c3d
MF		 2825 if (!mlx5e_vxlan_allowed(priv->mdev))
2826 return;
2827
974c3f30 2828 mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
b3f63c3d
MF		 2829}
2830
2831static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
2832 struct sk_buff *skb,
2833 netdev_features_t features)
2834{
2835 struct udphdr *udph;
2836 u16 proto;
2837 u16 port = 0;
2838
2839 switch (vlan_get_protocol(skb)) {
2840 case htons(ETH_P_IP):
2841 proto = ip_hdr(skb)->protocol;
2842 break;
2843 case htons(ETH_P_IPV6):
2844 proto = ipv6_hdr(skb)->nexthdr;
2845 break;
2846 default:
2847 goto out;
2848 }
2849
2850 if (proto == IPPROTO_UDP) {
2851 udph = udp_hdr(skb);
2852 port = be16_to_cpu(udph->dest);
2853 }
2854
2855 /* Verify if UDP port is being offloaded by HW */
2856 if (port && mlx5e_vxlan_lookup_port(priv, port))
2857 return features;
2858
2859out:
2860 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
2861 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2862}
2863
2864static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
2865 struct net_device *netdev,
2866 netdev_features_t features)
2867{
2868 struct mlx5e_priv *priv = netdev_priv(netdev);
2869
2870 features = vlan_features_check(skb, features);
2871 features = vxlan_features_check(skb, features);
2872
2873 /* Validate if the tunneled packet is being offloaded by HW */
2874 if (skb->encapsulation &&
2875 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
2876 return mlx5e_vxlan_features_check(priv, skb, features);
2877
2878 return features;
2879}
2880
3947ca18
DJ		 2881static void mlx5e_tx_timeout(struct net_device *dev)
2882{
2883 struct mlx5e_priv *priv = netdev_priv(dev);
2884 bool sched_work = false;
2885 int i;
2886
2887 netdev_err(dev, "TX timeout detected\n");
2888
2889 for (i = 0; i < priv->params.num_channels * priv->params.num_tc; i++) {
2890 struct mlx5e_sq *sq = priv->txq_to_sq_map[i];
2891
2c1ccc99 2892 if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i)))
3947ca18
DJ		 2893 continue;
2894 sched_work = true;
6e8dd6d6 2895 set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
3947ca18
DJ		 2896 netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n",
2897 i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc);
2898 }
2899
2900 if (sched_work && test_bit(MLX5E_STATE_OPENED, &priv->state))
2901 schedule_work(&priv->tx_timeout_work);
2902}
2903
b0eed40e 2904static const struct net_device_ops mlx5e_netdev_ops_basic = {
f62b8bb8
AV		 2905 .ndo_open = mlx5e_open,
2906 .ndo_stop = mlx5e_close,
2907 .ndo_start_xmit = mlx5e_xmit,
08fb1dac
SM		 2908 .ndo_setup_tc = mlx5e_ndo_setup_tc,
2909 .ndo_select_queue = mlx5e_select_queue,
f62b8bb8
AV		 2910 .ndo_get_stats64 = mlx5e_get_stats,
2911 .ndo_set_rx_mode = mlx5e_set_rx_mode,
2912 .ndo_set_mac_address = mlx5e_set_mac,
b0eed40e
SM		 2913 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid,
2914 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid,
f62b8bb8 2915 .ndo_set_features = mlx5e_set_features,
b0eed40e
SM		 2916 .ndo_change_mtu = mlx5e_change_mtu,
2917 .ndo_do_ioctl = mlx5e_ioctl,
507f0c81 2918 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate,
45bf454a
MG		 2919#ifdef CONFIG_RFS_ACCEL
2920 .ndo_rx_flow_steer = mlx5e_rx_flow_steer,
2921#endif
3947ca18 2922 .ndo_tx_timeout = mlx5e_tx_timeout,
b0eed40e
SM		 2923};
2924
2925static const struct net_device_ops mlx5e_netdev_ops_sriov = {
2926 .ndo_open = mlx5e_open,
2927 .ndo_stop = mlx5e_close,
2928 .ndo_start_xmit = mlx5e_xmit,
08fb1dac
SM		 2929 .ndo_setup_tc = mlx5e_ndo_setup_tc,
2930 .ndo_select_queue = mlx5e_select_queue,
b0eed40e
SM		 2931 .ndo_get_stats64 = mlx5e_get_stats,
2932 .ndo_set_rx_mode = mlx5e_set_rx_mode,
2933 .ndo_set_mac_address = mlx5e_set_mac,
2934 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid,
2935 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid,
2936 .ndo_set_features = mlx5e_set_features,
2937 .ndo_change_mtu = mlx5e_change_mtu,
2938 .ndo_do_ioctl = mlx5e_ioctl,
974c3f30
AD		 2939 .ndo_udp_tunnel_add = mlx5e_add_vxlan_port,
2940 .ndo_udp_tunnel_del = mlx5e_del_vxlan_port,
507f0c81 2941 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate,
b3f63c3d 2942 .ndo_features_check = mlx5e_features_check,
45bf454a
MG		 2943#ifdef CONFIG_RFS_ACCEL
2944 .ndo_rx_flow_steer = mlx5e_rx_flow_steer,
2945#endif
b0eed40e
SM		 2946 .ndo_set_vf_mac = mlx5e_set_vf_mac,
2947 .ndo_set_vf_vlan = mlx5e_set_vf_vlan,
f942380c 2948 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk,
1edc57e2 2949 .ndo_set_vf_trust = mlx5e_set_vf_trust,
b0eed40e
SM		 2950 .ndo_get_vf_config = mlx5e_get_vf_config,
2951 .ndo_set_vf_link_state = mlx5e_set_vf_link_state,
2952 .ndo_get_vf_stats = mlx5e_get_vf_stats,
3947ca18 2953 .ndo_tx_timeout = mlx5e_tx_timeout,
f62b8bb8
AV		 2954};
2955
2956static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
2957{
2958 if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
2959 return -ENOTSUPP;
2960 if (!MLX5_CAP_GEN(mdev, eth_net_offloads) ||
2961 !MLX5_CAP_GEN(mdev, nic_flow_table) ||
2962 !MLX5_CAP_ETH(mdev, csum_cap) ||
2963 !MLX5_CAP_ETH(mdev, max_lso_cap) ||
2964 !MLX5_CAP_ETH(mdev, vlan_cap) ||
796a27ec
GP		 2965 !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) ||
2966 MLX5_CAP_FLOWTABLE(mdev,
2967 flow_table_properties_nic_receive.max_ft_level)
2968 < 3) {
f62b8bb8
AV		 2969 mlx5_core_warn(mdev,
2970 "Not creating net device, some required device capabilities are missing\n");
2971 return -ENOTSUPP;
2972 }
66189961
TT		 2973 if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable))
2974 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n");
7524a5d8
GP		 2975 if (!MLX5_CAP_GEN(mdev, cq_moderation))
2976 mlx5_core_warn(mdev, "CQ modiration is not supported\n");
66189961 2977
f62b8bb8
AV		 2978 return 0;
2979}
2980
58d52291
AS		 2981u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
2982{
2983 int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
2984
2985 return bf_buf_size -
2986 sizeof(struct mlx5e_tx_wqe) +
2987 2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
2988}
2989
08fb1dac
SM		 2990#ifdef CONFIG_MLX5_CORE_EN_DCB
2991static void mlx5e_ets_init(struct mlx5e_priv *priv)
2992{
2993 int i;
2994
2995 priv->params.ets.ets_cap = mlx5_max_tc(priv->mdev) + 1;
2996 for (i = 0; i < priv->params.ets.ets_cap; i++) {
2997 priv->params.ets.tc_tx_bw[i] = MLX5E_MAX_BW_ALLOC;
2998 priv->params.ets.tc_tsa[i] = IEEE_8021QAZ_TSA_VENDOR;
2999 priv->params.ets.prio_tc[i] = i;
3000 }
3001
3002 /* tclass[prio=0]=1, tclass[prio=1]=0, tclass[prio=i]=i (for i>1) */
3003 priv->params.ets.prio_tc[0] = 1;
3004 priv->params.ets.prio_tc[1] = 0;
3005}
3006#endif
3007
d8c9660d
TT		 3008void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
3009 u32 *indirection_rqt, int len,
85082dba
TT		 3010 int num_channels)
3011{
d8c9660d
TT		 3012 int node = mdev->priv.numa_node;
3013 int node_num_of_cores;
85082dba
TT		 3014 int i;
3015
d8c9660d
TT		 3016 if (node == -1)
3017 node = first_online_node;
3018
3019 node_num_of_cores = cpumask_weight(cpumask_of_node(node));
3020
3021 if (node_num_of_cores)
3022 num_channels = min_t(int, num_channels, node_num_of_cores);
3023
85082dba
TT		 3024 for (i = 0; i < len; i++)
3025 indirection_rqt[i] = i % num_channels;
3026}
3027
bc77b240
TT		 3028static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
3029{
3030 return MLX5_CAP_GEN(mdev, striding_rq) &&
3031 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
3032 MLX5_CAP_ETH(mdev, reg_umr_sq);
3033}
3034
b797a684
SM		 3035static int mlx5e_get_pci_bw(struct mlx5_core_dev *mdev, u32 *pci_bw)
3036{
3037 enum pcie_link_width width;
3038 enum pci_bus_speed speed;
3039 int err = 0;
3040
3041 err = pcie_get_minimum_link(mdev->pdev, &speed, &width);
3042 if (err)
3043 return err;
3044
3045 if (speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
3046 return -EINVAL;
3047
3048 switch (speed) {
3049 case PCIE_SPEED_2_5GT:
3050 *pci_bw = 2500 * width;
3051 break;
3052 case PCIE_SPEED_5_0GT:
3053 *pci_bw = 5000 * width;
3054 break;
3055 case PCIE_SPEED_8_0GT:
3056 *pci_bw = 8000 * width;
3057 break;
3058 default:
3059 return -EINVAL;
3060 }
3061
3062 return 0;
3063}
3064
3065static bool cqe_compress_heuristic(u32 link_speed, u32 pci_bw)
3066{
3067 return (link_speed && pci_bw &&
3068 (pci_bw < 40000) && (pci_bw < link_speed));
3069}
3070
9908aa29
TT		 3071void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
3072{
3073 params->rx_cq_period_mode = cq_period_mode;
3074
3075 params->rx_cq_moderation.pkts =
3076 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3077 params->rx_cq_moderation.usec =
3078 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3079
3080 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
3081 params->rx_cq_moderation.usec =
3082 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
3083}
3084
cff92d7c
HHZ		 3085static void mlx5e_query_min_inline(struct mlx5_core_dev *mdev,
3086 u8 *min_inline_mode)
3087{
3088 switch (MLX5_CAP_ETH(mdev, wqe_inline_mode)) {
3089 case MLX5E_INLINE_MODE_L2:
3090 *min_inline_mode = MLX5_INLINE_MODE_L2;
3091 break;
3092 case MLX5E_INLINE_MODE_VPORT_CONTEXT:
3093 mlx5_query_nic_vport_min_inline(mdev,
3094 min_inline_mode);
3095 break;
3096 case MLX5_INLINE_MODE_NOT_REQUIRED:
3097 *min_inline_mode = MLX5_INLINE_MODE_NONE;
3098 break;
3099 }
3100}
3101
6bfd390b
HHZ		 3102static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
3103 struct net_device *netdev,
127ea380
HHZ		 3104 const struct mlx5e_profile *profile,
3105 void *ppriv)
f62b8bb8
AV		 3106{
3107 struct mlx5e_priv *priv = netdev_priv(netdev);
b797a684
SM		 3108 u32 link_speed = 0;
3109 u32 pci_bw = 0;
cb3c7fd4
GR		 3110 u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3111 MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
3112 MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
f62b8bb8
AV		 3113
3114 priv->params.log_sq_size =
3115 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
bc77b240 3116 priv->params.rq_wq_type = mlx5e_check_fragmented_striding_rq_cap(mdev) ?
461017cb
TT		 3117 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
3118 MLX5_WQ_TYPE_LINKED_LIST;
3119
b797a684
SM		 3120 /* set CQE compression */
3121 priv->params.rx_cqe_compress_admin = false;
3122 if (MLX5_CAP_GEN(mdev, cqe_compression) &&
3123 MLX5_CAP_GEN(mdev, vport_group_manager)) {
3124 mlx5e_get_max_linkspeed(mdev, &link_speed);
3125 mlx5e_get_pci_bw(mdev, &pci_bw);
3126 mlx5_core_dbg(mdev, "Max link speed = %d, PCI BW = %d\n",
3127 link_speed, pci_bw);
3128 priv->params.rx_cqe_compress_admin =
3129 cqe_compress_heuristic(link_speed, pci_bw);
3130 }
3131
3132 priv->params.rx_cqe_compress = priv->params.rx_cqe_compress_admin;
3133
461017cb
TT		 3134 switch (priv->params.rq_wq_type) {
3135 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
3136 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
d9d9f156
TT		 3137 priv->params.mpwqe_log_stride_sz =
3138 priv->params.rx_cqe_compress ?
3139 MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS :
3140 MLX5_MPWRQ_LOG_STRIDE_SIZE;
3141 priv->params.mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
3142 priv->params.mpwqe_log_stride_sz;
461017cb
TT		 3143 priv->params.lro_en = true;
3144 break;
3145 default: /* MLX5_WQ_TYPE_LINKED_LIST */
3146 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
3147 }
3148
d9d9f156
TT		 3149 mlx5_core_info(mdev,
3150 "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
3151 priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
3152 BIT(priv->params.log_rq_size),
3153 BIT(priv->params.mpwqe_log_stride_sz),
3154 priv->params.rx_cqe_compress_admin);
3155
461017cb
TT		 3156 priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
3157 BIT(priv->params.log_rq_size));
9908aa29 3158
cb3c7fd4
GR		 3159 priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
3160 mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
9908aa29
TT		 3161
3162 priv->params.tx_cq_moderation.usec =
f62b8bb8 3163 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
9908aa29 3164 priv->params.tx_cq_moderation.pkts =
f62b8bb8 3165 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
58d52291 3166 priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
cff92d7c 3167 mlx5e_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
f62b8bb8 3168 priv->params.num_tc = 1;
2be6967c 3169 priv->params.rss_hfunc = ETH_RSS_HASH_XOR;
f62b8bb8 3170
57afead5
AS		 3171 netdev_rss_key_fill(priv->params.toeplitz_hash_key,
3172 sizeof(priv->params.toeplitz_hash_key));
3173
d8c9660d 3174 mlx5e_build_default_indir_rqt(mdev, priv->params.indirection_rqt,
6bfd390b 3175 MLX5E_INDIR_RQT_SIZE, profile->max_nch(mdev));
2d75b2bc 3176
f62b8bb8
AV		 3177 priv->params.lro_wqe_sz =
3178 MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
3179
9908aa29
TT		 3180 /* Initialize pflags */
3181 MLX5E_SET_PRIV_FLAG(priv, MLX5E_PFLAG_RX_CQE_BASED_MODER,
3182 priv->params.rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
3183
f62b8bb8
AV		 3184 priv->mdev = mdev;
3185 priv->netdev = netdev;
6bfd390b
HHZ		 3186 priv->params.num_channels = profile->max_nch(mdev);
3187 priv->profile = profile;
127ea380 3188 priv->ppriv = ppriv;
f62b8bb8 3189
08fb1dac
SM		 3190#ifdef CONFIG_MLX5_CORE_EN_DCB
3191 mlx5e_ets_init(priv);
3192#endif
f62b8bb8 3193
f62b8bb8
AV		 3194 mutex_init(&priv->state_lock);
3195
3196 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3197 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3947ca18 3198 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
f62b8bb8
AV		 3199 INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3200}
3201
3202static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
3203{
3204 struct mlx5e_priv *priv = netdev_priv(netdev);
3205
e1d7d349 3206 mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr);
108805fc
SM		 3207 if (is_zero_ether_addr(netdev->dev_addr) &&
3208 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
3209 eth_hw_addr_random(netdev);
3210 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
3211 }
f62b8bb8
AV		 3212}
3213
cb67b832
HHZ		 3214static const struct switchdev_ops mlx5e_switchdev_ops = {
3215 .switchdev_port_attr_get = mlx5e_attr_get,
3216};
3217
6bfd390b 3218static void mlx5e_build_nic_netdev(struct net_device *netdev)
f62b8bb8
AV		 3219{
3220 struct mlx5e_priv *priv = netdev_priv(netdev);
3221 struct mlx5_core_dev *mdev = priv->mdev;
94cb1ebb
EBE		 3222 bool fcs_supported;
3223 bool fcs_enabled;
f62b8bb8
AV		 3224
3225 SET_NETDEV_DEV(netdev, &mdev->pdev->dev);
3226
08fb1dac 3227 if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
b0eed40e 3228 netdev->netdev_ops = &mlx5e_netdev_ops_sriov;
08fb1dac
SM		 3229#ifdef CONFIG_MLX5_CORE_EN_DCB
3230 netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
3231#endif
3232 } else {
b0eed40e 3233 netdev->netdev_ops = &mlx5e_netdev_ops_basic;
08fb1dac 3234 }
66e49ded 3235
f62b8bb8
AV		 3236 netdev->watchdog_timeo = 15 * HZ;
3237
3238 netdev->ethtool_ops = &mlx5e_ethtool_ops;
3239
12be4b21 3240 netdev->vlan_features |= NETIF_F_SG;
f62b8bb8
AV		 3241 netdev->vlan_features |= NETIF_F_IP_CSUM;
3242 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3243 netdev->vlan_features |= NETIF_F_GRO;
3244 netdev->vlan_features |= NETIF_F_TSO;
3245 netdev->vlan_features |= NETIF_F_TSO6;
3246 netdev->vlan_features |= NETIF_F_RXCSUM;
3247 netdev->vlan_features |= NETIF_F_RXHASH;
3248
3249 if (!!MLX5_CAP_ETH(mdev, lro_cap))
3250 netdev->vlan_features |= NETIF_F_LRO;
3251
3252 netdev->hw_features = netdev->vlan_features;
e4cf27bd 3253 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
f62b8bb8
AV		 3254 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
3255 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
3256
b3f63c3d 3257 if (mlx5e_vxlan_allowed(mdev)) {
b49663c8
AD		 3258 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
3259 NETIF_F_GSO_UDP_TUNNEL_CSUM |
3260 NETIF_F_GSO_PARTIAL;
b3f63c3d 3261 netdev->hw_enc_features |= NETIF_F_IP_CSUM;
f3ed653c 3262 netdev->hw_enc_features |= NETIF_F_IPV6_CSUM;
b3f63c3d
MF		 3263 netdev->hw_enc_features |= NETIF_F_TSO;
3264 netdev->hw_enc_features |= NETIF_F_TSO6;
b3f63c3d 3265 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
b49663c8
AD		 3266 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
3267 NETIF_F_GSO_PARTIAL;
3268 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
b3f63c3d
MF		 3269 }
3270
94cb1ebb
EBE		 3271 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
3272
3273 if (fcs_supported)
3274 netdev->hw_features |= NETIF_F_RXALL;
3275
f62b8bb8
AV		 3276 netdev->features = netdev->hw_features;
3277 if (!priv->params.lro_en)
3278 netdev->features &= ~NETIF_F_LRO;
3279
94cb1ebb
EBE		 3280 if (fcs_enabled)
3281 netdev->features &= ~NETIF_F_RXALL;
3282
e8f887ac
AV		 3283#define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
3284 if (FT_CAP(flow_modify_en) &&
3285 FT_CAP(modify_root) &&
3286 FT_CAP(identified_miss_table_mode) &&
1cabe6b0
MG		 3287 FT_CAP(flow_table_modify)) {
3288 netdev->hw_features |= NETIF_F_HW_TC;
3289#ifdef CONFIG_RFS_ACCEL
3290 netdev->hw_features |= NETIF_F_NTUPLE;
3291#endif
3292 }
e8f887ac 3293
f62b8bb8
AV		 3294 netdev->features |= NETIF_F_HIGHDMA;
3295
3296 netdev->priv_flags |= IFF_UNICAST_FLT;
3297
3298 mlx5e_set_netdev_dev_addr(netdev);
cb67b832
HHZ		 3299
3300#ifdef CONFIG_NET_SWITCHDEV
3301 if (MLX5_CAP_GEN(mdev, vport_group_manager))
3302 netdev->switchdev_ops = &mlx5e_switchdev_ops;
3303#endif
f62b8bb8
AV		 3304}
3305
593cf338
RS		 3306static void mlx5e_create_q_counter(struct mlx5e_priv *priv)
3307{
3308 struct mlx5_core_dev *mdev = priv->mdev;
3309 int err;
3310
3311 err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter);
3312 if (err) {
3313 mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err);
3314 priv->q_counter = 0;
3315 }
3316}
3317
3318static void mlx5e_destroy_q_counter(struct mlx5e_priv *priv)
3319{
3320 if (!priv->q_counter)
3321 return;
3322
3323 mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter);
3324}
3325
bc77b240
TT		 3326static int mlx5e_create_umr_mkey(struct mlx5e_priv *priv)
3327{
3328 struct mlx5_core_dev *mdev = priv->mdev;
fe4c988b
SM		 3329 u64 npages = MLX5E_REQUIRED_MTTS(priv->profile->max_nch(mdev),
3330 BIT(MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW));
ec22eb53
SM		 3331 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3332 void *mkc;
3333 u32 *in;
bc77b240
TT		 3334 int err;
3335
3336 in = mlx5_vzalloc(inlen);
3337 if (!in)
3338 return -ENOMEM;
3339
ec22eb53 3340 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
bc77b240 3341
fe4c988b
SM		 3342 npages = min_t(u32, ALIGN(U16_MAX, 4) * 2, npages);
3343
ec22eb53
SM		 3344 MLX5_SET(mkc, mkc, free, 1);
3345 MLX5_SET(mkc, mkc, umr_en, 1);
3346 MLX5_SET(mkc, mkc, lw, 1);
3347 MLX5_SET(mkc, mkc, lr, 1);
3348 MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
bc77b240 3349
ec22eb53
SM		 3350 MLX5_SET(mkc, mkc, qpn, 0xffffff);
3351 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
3352 MLX5_SET64(mkc, mkc, len, npages << PAGE_SHIFT);
3353 MLX5_SET(mkc, mkc, translations_octword_size,
6abdd5f5 3354 MLX5_MTT_OCTW(npages));
ec22eb53 3355 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
bc77b240 3356
ec22eb53 3357 err = mlx5_core_create_mkey(mdev, &priv->umr_mkey, in, inlen);
bc77b240 3358
ec22eb53 3359 kvfree(in);
bc77b240
TT		 3360 return err;
3361}
3362
6bfd390b
HHZ		 3363static void mlx5e_nic_init(struct mlx5_core_dev *mdev,
3364 struct net_device *netdev,
127ea380
HHZ		 3365 const struct mlx5e_profile *profile,
3366 void *ppriv)
6bfd390b
HHZ		 3367{
3368 struct mlx5e_priv *priv = netdev_priv(netdev);
3369
127ea380 3370 mlx5e_build_nic_netdev_priv(mdev, netdev, profile, ppriv);
6bfd390b
HHZ		 3371 mlx5e_build_nic_netdev(netdev);
3372 mlx5e_vxlan_init(priv);
3373}
3374
3375static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
3376{
127ea380
HHZ		 3377 struct mlx5_core_dev *mdev = priv->mdev;
3378 struct mlx5_eswitch *esw = mdev->priv.eswitch;
3379
6bfd390b 3380 mlx5e_vxlan_cleanup(priv);
127ea380
HHZ		 3381
3382 if (MLX5_CAP_GEN(mdev, vport_group_manager))
3383 mlx5_eswitch_unregister_vport_rep(esw, 0);
6bfd390b
HHZ		 3384}
3385
3386static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
3387{
3388 struct mlx5_core_dev *mdev = priv->mdev;
3389 int err;
3390 int i;
3391
3392 err = mlx5e_create_indirect_rqts(priv);
3393 if (err) {
3394 mlx5_core_warn(mdev, "create indirect rqts failed, %d\n", err);
3395 return err;
3396 }
3397
3398 err = mlx5e_create_direct_rqts(priv);
3399 if (err) {
3400 mlx5_core_warn(mdev, "create direct rqts failed, %d\n", err);
3401 goto err_destroy_indirect_rqts;
3402 }
3403
3404 err = mlx5e_create_indirect_tirs(priv);
3405 if (err) {
3406 mlx5_core_warn(mdev, "create indirect tirs failed, %d\n", err);
3407 goto err_destroy_direct_rqts;
3408 }
3409
3410 err = mlx5e_create_direct_tirs(priv);
3411 if (err) {
3412 mlx5_core_warn(mdev, "create direct tirs failed, %d\n", err);
3413 goto err_destroy_indirect_tirs;
3414 }
3415
3416 err = mlx5e_create_flow_steering(priv);
3417 if (err) {
3418 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
3419 goto err_destroy_direct_tirs;
3420 }
3421
3422 err = mlx5e_tc_init(priv);
3423 if (err)
3424 goto err_destroy_flow_steering;
3425
3426 return 0;
3427
3428err_destroy_flow_steering:
3429 mlx5e_destroy_flow_steering(priv);
3430err_destroy_direct_tirs:
3431 mlx5e_destroy_direct_tirs(priv);
3432err_destroy_indirect_tirs:
3433 mlx5e_destroy_indirect_tirs(priv);
3434err_destroy_direct_rqts:
3435 for (i = 0; i < priv->profile->max_nch(mdev); i++)
3436 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
3437err_destroy_indirect_rqts:
3438 mlx5e_destroy_rqt(priv, &priv->indir_rqt);
3439 return err;
3440}
3441
3442static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
3443{
3444 int i;
3445
3446 mlx5e_tc_cleanup(priv);
3447 mlx5e_destroy_flow_steering(priv);
3448 mlx5e_destroy_direct_tirs(priv);
3449 mlx5e_destroy_indirect_tirs(priv);
3450 for (i = 0; i < priv->profile->max_nch(priv->mdev); i++)
3451 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
3452 mlx5e_destroy_rqt(priv, &priv->indir_rqt);
3453}
3454
3455static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
3456{
3457 int err;
3458
3459 err = mlx5e_create_tises(priv);
3460 if (err) {
3461 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
3462 return err;
3463 }
3464
3465#ifdef CONFIG_MLX5_CORE_EN_DCB
3466 mlx5e_dcbnl_ieee_setets_core(priv, &priv->params.ets);
3467#endif
3468 return 0;
3469}
3470
3471static void mlx5e_nic_enable(struct mlx5e_priv *priv)
3472{
3473 struct net_device *netdev = priv->netdev;
3474 struct mlx5_core_dev *mdev = priv->mdev;
127ea380
HHZ		 3475 struct mlx5_eswitch *esw = mdev->priv.eswitch;
3476 struct mlx5_eswitch_rep rep;
6bfd390b 3477
7907f23a
AH		 3478 mlx5_lag_add(mdev, netdev);
3479
6bfd390b
HHZ		 3480 if (mlx5e_vxlan_allowed(mdev)) {
3481 rtnl_lock();
3482 udp_tunnel_get_rx_info(netdev);
3483 rtnl_unlock();
3484 }
3485
3486 mlx5e_enable_async_events(priv);
3487 queue_work(priv->wq, &priv->set_rx_mode_work);
127ea380
HHZ		 3488
3489 if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
dbe413e3 3490 mlx5_query_nic_vport_mac_address(mdev, 0, rep.hw_id);
cb67b832
HHZ		 3491 rep.load = mlx5e_nic_rep_load;
3492 rep.unload = mlx5e_nic_rep_unload;
127ea380
HHZ		 3493 rep.vport = 0;
3494 rep.priv_data = priv;
3495 mlx5_eswitch_register_vport_rep(esw, &rep);
3496 }
6bfd390b
HHZ		 3497}
3498
3499static void mlx5e_nic_disable(struct mlx5e_priv *priv)
3500{
3501 queue_work(priv->wq, &priv->set_rx_mode_work);
3502 mlx5e_disable_async_events(priv);
7907f23a 3503 mlx5_lag_remove(priv->mdev);
6bfd390b
HHZ		 3504}
3505
3506static const struct mlx5e_profile mlx5e_nic_profile = {
3507 .init = mlx5e_nic_init,
3508 .cleanup = mlx5e_nic_cleanup,
3509 .init_rx = mlx5e_init_nic_rx,
3510 .cleanup_rx = mlx5e_cleanup_nic_rx,
3511 .init_tx = mlx5e_init_nic_tx,
3512 .cleanup_tx = mlx5e_cleanup_nic_tx,
3513 .enable = mlx5e_nic_enable,
3514 .disable = mlx5e_nic_disable,
3515 .update_stats = mlx5e_update_stats,
3516 .max_nch = mlx5e_get_max_num_channels,
3517 .max_tc = MLX5E_MAX_NUM_TC,
3518};
3519
26e59d80
MHY		 3520struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
3521 const struct mlx5e_profile *profile,
3522 void *ppriv)
f62b8bb8 3523{
26e59d80 3524 int nch = profile->max_nch(mdev);
f62b8bb8
AV		 3525 struct net_device *netdev;
3526 struct mlx5e_priv *priv;
f62b8bb8 3527
08fb1dac 3528 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
6bfd390b 3529 nch * profile->max_tc,
08fb1dac 3530 nch);
f62b8bb8
AV		 3531 if (!netdev) {
3532 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
3533 return NULL;
3534 }
3535
127ea380 3536 profile->init(mdev, netdev, profile, ppriv);
f62b8bb8
AV		 3537
3538 netif_carrier_off(netdev);
3539
3540 priv = netdev_priv(netdev);
3541
7bb29755
MF		 3542 priv->wq = create_singlethread_workqueue("mlx5e");
3543 if (!priv->wq)
26e59d80
MHY		 3544 goto err_cleanup_nic;
3545
3546 return netdev;
3547
3548err_cleanup_nic:
3549 profile->cleanup(priv);
3550 free_netdev(netdev);
3551
3552 return NULL;
3553}
3554
3555int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
3556{
3557 const struct mlx5e_profile *profile;
3558 struct mlx5e_priv *priv;
3559 int err;
3560
3561 priv = netdev_priv(netdev);
3562 profile = priv->profile;
3563 clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
7bb29755 3564
bc77b240
TT		 3565 err = mlx5e_create_umr_mkey(priv);
3566 if (err) {
3567 mlx5_core_err(mdev, "create umr mkey failed, %d\n", err);
26e59d80 3568 goto out;
bc77b240
TT		 3569 }
3570
6bfd390b
HHZ		 3571 err = profile->init_tx(priv);
3572 if (err)
bc77b240 3573 goto err_destroy_umr_mkey;
5c50368f
AS		 3574
3575 err = mlx5e_open_drop_rq(priv);
3576 if (err) {
3577 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
6bfd390b 3578 goto err_cleanup_tx;
5c50368f
AS		 3579 }
3580
6bfd390b
HHZ		 3581 err = profile->init_rx(priv);
3582 if (err)
5c50368f 3583 goto err_close_drop_rq;
5c50368f 3584
593cf338
RS		 3585 mlx5e_create_q_counter(priv);
3586
33cfaaa8 3587 mlx5e_init_l2_addr(priv);
5c50368f 3588
13f9bba7
SM		 3589 mlx5e_set_dev_port_mtu(netdev);
3590
6bfd390b
HHZ		 3591 if (profile->enable)
3592 profile->enable(priv);
f62b8bb8 3593
26e59d80
MHY		 3594 rtnl_lock();
3595 if (netif_running(netdev))
3596 mlx5e_open(netdev);
3597 netif_device_attach(netdev);
3598 rtnl_unlock();
f62b8bb8 3599
26e59d80 3600 return 0;
5c50368f
AS		 3601
3602err_close_drop_rq:
3603 mlx5e_close_drop_rq(priv);
3604
6bfd390b
HHZ		 3605err_cleanup_tx:
3606 profile->cleanup_tx(priv);
5c50368f 3607
bc77b240
TT		 3608err_destroy_umr_mkey:
3609 mlx5_core_destroy_mkey(mdev, &priv->umr_mkey);
3610
26e59d80
MHY		 3611out:
3612 return err;
f62b8bb8
AV		 3613}
3614
127ea380
HHZ		 3615static void mlx5e_register_vport_rep(struct mlx5_core_dev *mdev)
3616{
3617 struct mlx5_eswitch *esw = mdev->priv.eswitch;
3618 int total_vfs = MLX5_TOTAL_VPORTS(mdev);
3619 int vport;
dbe413e3 3620 u8 mac[ETH_ALEN];
127ea380
HHZ		 3621
3622 if (!MLX5_CAP_GEN(mdev, vport_group_manager))
3623 return;
3624
dbe413e3
HHZ		 3625 mlx5_query_nic_vport_mac_address(mdev, 0, mac);
3626
127ea380
HHZ		 3627 for (vport = 1; vport < total_vfs; vport++) {
3628 struct mlx5_eswitch_rep rep;
3629
cb67b832
HHZ		 3630 rep.load = mlx5e_vport_rep_load;
3631 rep.unload = mlx5e_vport_rep_unload;
127ea380 3632 rep.vport = vport;
dbe413e3 3633 ether_addr_copy(rep.hw_id, mac);
127ea380
HHZ		 3634 mlx5_eswitch_register_vport_rep(esw, &rep);
3635 }
3636}
3637
26e59d80
MHY		 3638void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
3639{
3640 struct mlx5e_priv *priv = netdev_priv(netdev);
3641 const struct mlx5e_profile *profile = priv->profile;
3642
3643 set_bit(MLX5E_STATE_DESTROYING, &priv->state);
3644 if (profile->disable)
3645 profile->disable(priv);
3646
3647 flush_workqueue(priv->wq);
3648
3649 rtnl_lock();
3650 if (netif_running(netdev))
3651 mlx5e_close(netdev);
3652 netif_device_detach(netdev);
3653 rtnl_unlock();
3654
3655 mlx5e_destroy_q_counter(priv);
3656 profile->cleanup_rx(priv);
3657 mlx5e_close_drop_rq(priv);
3658 profile->cleanup_tx(priv);
3659 mlx5_core_destroy_mkey(priv->mdev, &priv->umr_mkey);
3660 cancel_delayed_work_sync(&priv->update_stats_work);
3661}
3662
3663/* mlx5e_attach and mlx5e_detach scope should be only creating/destroying
3664 * hardware contexts and to connect it to the current netdev.
3665 */
3666static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv)
3667{
3668 struct mlx5e_priv *priv = vpriv;
3669 struct net_device *netdev = priv->netdev;
3670 int err;
3671
3672 if (netif_device_present(netdev))
3673 return 0;
3674
3675 err = mlx5e_create_mdev_resources(mdev);
3676 if (err)
3677 return err;
3678
3679 err = mlx5e_attach_netdev(mdev, netdev);
3680 if (err) {
3681 mlx5e_destroy_mdev_resources(mdev);
3682 return err;
3683 }
3684
3685 return 0;
3686}
3687
3688static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv)
3689{
3690 struct mlx5e_priv *priv = vpriv;
3691 struct net_device *netdev = priv->netdev;
3692
3693 if (!netif_device_present(netdev))
3694 return;
3695
3696 mlx5e_detach_netdev(mdev, netdev);
3697 mlx5e_destroy_mdev_resources(mdev);
3698}
3699
b50d292b
HHZ		 3700static void *mlx5e_add(struct mlx5_core_dev *mdev)
3701{
127ea380 3702 struct mlx5_eswitch *esw = mdev->priv.eswitch;
26e59d80 3703 int total_vfs = MLX5_TOTAL_VPORTS(mdev);
127ea380 3704 void *ppriv = NULL;
26e59d80
MHY		 3705 void *priv;
3706 int vport;
3707 int err;
3708 struct net_device *netdev;
b50d292b 3709
26e59d80
MHY		 3710 err = mlx5e_check_required_hca_cap(mdev);
3711 if (err)
b50d292b
HHZ		 3712 return NULL;
3713
127ea380
HHZ		 3714 mlx5e_register_vport_rep(mdev);
3715
3716 if (MLX5_CAP_GEN(mdev, vport_group_manager))
3717 ppriv = &esw->offloads.vport_reps[0];
3718
26e59d80
MHY		 3719 netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, ppriv);
3720 if (!netdev) {
3721 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
3722 goto err_unregister_reps;
3723 }
3724
3725 priv = netdev_priv(netdev);
3726
3727 err = mlx5e_attach(mdev, priv);
3728 if (err) {
3729 mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err);
3730 goto err_destroy_netdev;
3731 }
3732
3733 err = register_netdev(netdev);
3734 if (err) {
3735 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
3736 goto err_detach;
b50d292b 3737 }
26e59d80
MHY		 3738
3739 return priv;
3740
3741err_detach:
3742 mlx5e_detach(mdev, priv);
3743
3744err_destroy_netdev:
3745 mlx5e_destroy_netdev(mdev, priv);
3746
3747err_unregister_reps:
3748 for (vport = 1; vport < total_vfs; vport++)
3749 mlx5_eswitch_unregister_vport_rep(esw, vport);
3750
3751 return NULL;
b50d292b
HHZ		 3752}
3753
cb67b832 3754void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv)
f62b8bb8 3755{
6bfd390b 3756 const struct mlx5e_profile *profile = priv->profile;
f62b8bb8
AV		 3757 struct net_device *netdev = priv->netdev;
3758
26e59d80 3759 unregister_netdev(netdev);
7bb29755 3760 destroy_workqueue(priv->wq);
6bfd390b
HHZ		 3761 if (profile->cleanup)
3762 profile->cleanup(priv);
26e59d80 3763 free_netdev(netdev);
f62b8bb8
AV		 3764}
3765
b50d292b
HHZ		 3766static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv)
3767{
127ea380
HHZ		 3768 struct mlx5_eswitch *esw = mdev->priv.eswitch;
3769 int total_vfs = MLX5_TOTAL_VPORTS(mdev);
b50d292b 3770 struct mlx5e_priv *priv = vpriv;
127ea380 3771 int vport;
b50d292b 3772
127ea380
HHZ		 3773 for (vport = 1; vport < total_vfs; vport++)
3774 mlx5_eswitch_unregister_vport_rep(esw, vport);
3775
26e59d80
MHY		 3776 mlx5e_detach(mdev, vpriv);
3777 mlx5e_destroy_netdev(mdev, priv);
b50d292b
HHZ		 3778}
3779
f62b8bb8
AV		 3780static void *mlx5e_get_netdev(void *vpriv)
3781{
3782 struct mlx5e_priv *priv = vpriv;
3783
3784 return priv->netdev;
3785}
3786
3787static struct mlx5_interface mlx5e_interface = {
b50d292b
HHZ		 3788 .add = mlx5e_add,
3789 .remove = mlx5e_remove,
26e59d80
MHY		 3790 .attach = mlx5e_attach,
3791 .detach = mlx5e_detach,
f62b8bb8
AV		 3792 .event = mlx5e_async_event,
3793 .protocol = MLX5_INTERFACE_PROTOCOL_ETH,
3794 .get_dev = mlx5e_get_netdev,
3795};
3796
3797void mlx5e_init(void)
3798{
665bc539 3799 mlx5e_build_ptys2ethtool_map();
f62b8bb8
AV		 3800 mlx5_register_interface(&mlx5e_interface);
3801}
3802
3803void mlx5e_cleanup(void)
3804{
3805 mlx5_unregister_interface(&mlx5e_interface);
3806}
Mirror https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git