]> git.ipfire.org Git - thirdparty/kernel/stable.git/blame - drivers/net/ethernet/mellanox/mlx5/core/en_main.c
projects / thirdparty / kernel / stable.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/mlx5e: Add counter for XDP redirect in RX
[thirdparty/kernel/stable.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_main.c
CommitLineData
f62b8bb8 1/*
b3f63c3d 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
f62b8bb8
AV		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
e8f887ac
AV		 33#include <net/tc_act/tc_gact.h>
34#include <net/pkt_cls.h>
86d722ad 35#include <linux/mlx5/fs.h>
b3f63c3d 36#include <net/vxlan.h>
86994156 37#include <linux/bpf.h>
60bbf7ee 38#include <net/page_pool.h>
1d447a39 39#include "eswitch.h"
f62b8bb8 40#include "en.h"
e8f887ac 41#include "en_tc.h"
1d447a39 42#include "en_rep.h"
547eede0 43#include "en_accel/ipsec.h"
899a59d3 44#include "en_accel/ipsec_rxtx.h"
c83294b9 45#include "en_accel/tls.h"
899a59d3 46#include "accel/ipsec.h"
c83294b9 47#include "accel/tls.h"
b3f63c3d 48#include "vxlan.h"
2c81bfd5 49#include "en/port.h"
159d2131 50#include "en/xdp.h"
f62b8bb8
AV		 51
52struct mlx5e_rq_param {
cb3c7fd4
GR		 53 u32 rqc[MLX5_ST_SZ_DW(rqc)];
54 struct mlx5_wq_param wq;
069d1146 55 struct mlx5e_rq_frags_info frags_info;
f62b8bb8
AV		 56};
57
58struct mlx5e_sq_param {
59 u32 sqc[MLX5_ST_SZ_DW(sqc)];
60 struct mlx5_wq_param wq;
61};
62
63struct mlx5e_cq_param {
64 u32 cqc[MLX5_ST_SZ_DW(cqc)];
65 struct mlx5_wq_param wq;
66 u16 eq_ix;
9908aa29 67 u8 cq_period_mode;
f62b8bb8
AV		 68};
69
70struct mlx5e_channel_param {
71 struct mlx5e_rq_param rq;
72 struct mlx5e_sq_param sq;
b5503b99 73 struct mlx5e_sq_param xdp_sq;
d3c9bc27 74 struct mlx5e_sq_param icosq;
f62b8bb8
AV		 75 struct mlx5e_cq_param rx_cq;
76 struct mlx5e_cq_param tx_cq;
d3c9bc27 77 struct mlx5e_cq_param icosq_cq;
f62b8bb8
AV		 78};
79
2ccb0a79 80bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
2fc4bfb7 81{
ea3886ca 82 bool striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) &&
2fc4bfb7
SM		 83 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
84 MLX5_CAP_ETH(mdev, reg_umr_sq);
ea3886ca
TT		 85 u16 max_wqe_sz_cap = MLX5_CAP_GEN(mdev, max_wqe_sz_sq);
86 bool inline_umr = MLX5E_UMR_WQE_INLINE_SZ <= max_wqe_sz_cap;
87
88 if (!striding_rq_umr)
89 return false;
90 if (!inline_umr) {
91 mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n",
92 (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap);
93 return false;
94 }
95 return true;
2fc4bfb7
SM		 96}
97
069d1146 98static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)
73281b78 99{
a26a5bdf
TT		 100 u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
101 u16 linear_rq_headroom = params->xdp_prog ?
102 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
103 u32 frag_sz;
73281b78 104
a26a5bdf 105 linear_rq_headroom += NET_IP_ALIGN;
619a8f2a 106
a26a5bdf
TT		 107 frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);
108
109 if (params->xdp_prog && frag_sz < PAGE_SIZE)
110 frag_sz = PAGE_SIZE;
111
112 return frag_sz;
73281b78
TT		 113}
114
115static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
116{
069d1146 117 u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params);
73281b78
TT		 118
119 return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
120}
121
069d1146
TT		 122static bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
123 struct mlx5e_params *params)
124{
125 u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
126
127 return !params->lro_en && frag_sz <= PAGE_SIZE;
128}
129
619a8f2a
TT		 130static bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
131 struct mlx5e_params *params)
132{
069d1146 133 u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
619a8f2a
TT		 134 s8 signed_log_num_strides_param;
135 u8 log_num_strides;
136
069d1146 137 if (!mlx5e_rx_is_linear_skb(mdev, params))
619a8f2a
TT		 138 return false;
139
140 if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
141 return true;
142
143 log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz);
144 signed_log_num_strides_param =
145 (s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
146
147 return signed_log_num_strides_param >= 0;
148}
149
73281b78
TT		 150static u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params)
151{
152 if (params->log_rq_mtu_frames <
153 mlx5e_mpwqe_log_pkts_per_wqe(params) + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
154 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
155
156 return params->log_rq_mtu_frames - mlx5e_mpwqe_log_pkts_per_wqe(params);
157}
158
159static u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
160 struct mlx5e_params *params)
f1e4fc9b 161{
619a8f2a 162 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
069d1146 163 return order_base_2(mlx5e_rx_get_linear_frag_sz(params));
619a8f2a 164
f1e4fc9b
TT		 165 return MLX5E_MPWQE_STRIDE_SZ(mdev,
166 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
167}
168
73281b78
TT		 169static u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
170 struct mlx5e_params *params)
f1e4fc9b
TT		 171{
172 return MLX5_MPWRQ_LOG_WQE_SZ -
173 mlx5e_mpwqe_get_log_stride_size(mdev, params);
174}
175
619a8f2a
TT		 176static u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
177 struct mlx5e_params *params)
b0cedc84
TT		 178{
179 u16 linear_rq_headroom = params->xdp_prog ?
180 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
069d1146 181 bool is_linear_skb;
b0cedc84
TT		 182
183 linear_rq_headroom += NET_IP_ALIGN;
184
069d1146
TT		 185 is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
186 mlx5e_rx_is_linear_skb(mdev, params) :
187 mlx5e_rx_mpwqe_is_linear_skb(mdev, params);
b0cedc84 188
069d1146 189 return is_linear_skb ? linear_rq_headroom : 0;
b0cedc84
TT		 190}
191
696a97cf 192void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
2a0f561b 193 struct mlx5e_params *params)
2fc4bfb7 194{
6a9764ef 195 params->lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
73281b78
TT		 196 params->log_rq_mtu_frames = is_kdump_kernel() ?
197 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
198 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
2fc4bfb7 199
6a9764ef
SM		 200 mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
201 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
619a8f2a
TT		 202 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?
203 BIT(mlx5e_mpwqe_get_log_rq_size(params)) :
73281b78 204 BIT(params->log_rq_mtu_frames),
f1e4fc9b 205 BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params)),
6a9764ef 206 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
2fc4bfb7
SM		 207}
208
2ccb0a79
TT		 209bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
210 struct mlx5e_params *params)
211{
212 return mlx5e_check_fragmented_striding_rq_cap(mdev) &&
22f45398
TT		 213 !MLX5_IPSEC_DEV(mdev) &&
214 !(params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params));
2ccb0a79 215}
291f445e 216
2ccb0a79 217void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
2fc4bfb7 218{
2ccb0a79
TT		 219 params->rq_wq_type = mlx5e_striding_rq_possible(mdev, params) &&
220 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ?
291f445e 221 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
99cbfa93 222 MLX5_WQ_TYPE_CYCLIC;
2fc4bfb7
SM		 223}
224
f62b8bb8
AV		 225static void mlx5e_update_carrier(struct mlx5e_priv *priv)
226{
227 struct mlx5_core_dev *mdev = priv->mdev;
228 u8 port_state;
229
230 port_state = mlx5_query_vport_state(mdev,
e53eef63
OG		 231 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT,
232 0);
f62b8bb8 233
87424ad5
SD		 234 if (port_state == VPORT_STATE_UP) {
235 netdev_info(priv->netdev, "Link up\n");
f62b8bb8 236 netif_carrier_on(priv->netdev);
87424ad5
SD		 237 } else {
238 netdev_info(priv->netdev, "Link down\n");
f62b8bb8 239 netif_carrier_off(priv->netdev);
87424ad5 240 }
f62b8bb8
AV		 241}
242
243static void mlx5e_update_carrier_work(struct work_struct *work)
244{
245 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
246 update_carrier_work);
247
248 mutex_lock(&priv->state_lock);
249 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
7ca42c80
ES		 250 if (priv->profile->update_carrier)
251 priv->profile->update_carrier(priv);
f62b8bb8
AV		 252 mutex_unlock(&priv->state_lock);
253}
254
19386177 255void mlx5e_update_stats(struct mlx5e_priv *priv)
f62b8bb8 256{
19386177 257 int i;
f62b8bb8 258
19386177
KH		 259 for (i = mlx5e_num_stats_grps - 1; i >= 0; i--)
260 if (mlx5e_stats_grps[i].update_stats)
261 mlx5e_stats_grps[i].update_stats(priv);
f62b8bb8
AV		 262}
263
3834a5e6
GP		 264static void mlx5e_update_ndo_stats(struct mlx5e_priv *priv)
265{
19386177
KH		 266 int i;
267
268 for (i = mlx5e_num_stats_grps - 1; i >= 0; i--)
269 if (mlx5e_stats_grps[i].update_stats_mask &
270 MLX5E_NDO_UPDATE_STATS)
271 mlx5e_stats_grps[i].update_stats(priv);
3834a5e6
GP		 272}
273
cb67b832 274void mlx5e_update_stats_work(struct work_struct *work)
f62b8bb8
AV		 275{
276 struct delayed_work *dwork = to_delayed_work(work);
277 struct mlx5e_priv *priv = container_of(dwork, struct mlx5e_priv,
278 update_stats_work);
ed56c519 279
f62b8bb8 280 mutex_lock(&priv->state_lock);
ed56c519 281 priv->profile->update_stats(priv);
f62b8bb8
AV		 282 mutex_unlock(&priv->state_lock);
283}
284
daa21560
TT		 285static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
286 enum mlx5_dev_event event, unsigned long param)
f62b8bb8 287{
daa21560
TT		 288 struct mlx5e_priv *priv = vpriv;
289
e0f46eb9 290 if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state))
daa21560
TT		 291 return;
292
f62b8bb8
AV		 293 switch (event) {
294 case MLX5_DEV_EVENT_PORT_UP:
295 case MLX5_DEV_EVENT_PORT_DOWN:
7bb29755 296 queue_work(priv->wq, &priv->update_carrier_work);
f62b8bb8 297 break;
f62b8bb8
AV		 298 default:
299 break;
300 }
301}
302
f62b8bb8
AV		 303static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
304{
e0f46eb9 305 set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
f62b8bb8
AV		 306}
307
308static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
309{
e0f46eb9 310 clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
78249c42 311 synchronize_irq(pci_irq_vector(priv->mdev->pdev, MLX5_EQ_VEC_ASYNC));
f62b8bb8
AV		 312}
313
31391048
SM		 314static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
315 struct mlx5e_icosq *sq,
b8a98a4c 316 struct mlx5e_umr_wqe *wqe)
7e426671
TT		 317{
318 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
319 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
ea3886ca 320 u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS);
7e426671
TT		 321
322 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
323 ds_cnt);
324 cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
325 cseg->imm = rq->mkey_be;
326
ea3886ca 327 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
31616255 328 ucseg->xlt_octowords =
7e426671 329 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
7e426671 330 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
7e426671
TT		 331}
332
422d4c40
TT		 333static u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq)
334{
335 switch (rq->wq_type) {
336 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
337 return mlx5_wq_ll_get_size(&rq->mpwqe.wq);
338 default:
99cbfa93 339 return mlx5_wq_cyc_get_size(&rq->wqe.wq);
422d4c40
TT		 340 }
341}
342
343static u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq)
344{
345 switch (rq->wq_type) {
346 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
347 return rq->mpwqe.wq.cur_sz;
348 default:
349 return rq->wqe.wq.cur_sz;
350 }
351}
352
7e426671
TT		 353static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
354 struct mlx5e_channel *c)
355{
422d4c40 356 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
7e426671 357
eec4edc9
KC		 358 rq->mpwqe.info = kvzalloc_node(array_size(wq_sz,
359 sizeof(*rq->mpwqe.info)),
ca11b798 360 GFP_KERNEL, cpu_to_node(c->cpu));
21c59685 361 if (!rq->mpwqe.info)
ea3886ca 362 return -ENOMEM;
7e426671 363
b8a98a4c 364 mlx5e_build_umr_wqe(rq, &c->icosq, &rq->mpwqe.umr_wqe);
7e426671
TT		 365
366 return 0;
7e426671
TT		 367}
368
a43b25da 369static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
ec8b9981
TT		 370 u64 npages, u8 page_shift,
371 struct mlx5_core_mkey *umr_mkey)
3608ae77 372{
3608ae77
TT		 373 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
374 void *mkc;
375 u32 *in;
376 int err;
377
1b9a07ee 378 in = kvzalloc(inlen, GFP_KERNEL);
3608ae77
TT		 379 if (!in)
380 return -ENOMEM;
381
382 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
383
3608ae77
TT		 384 MLX5_SET(mkc, mkc, free, 1);
385 MLX5_SET(mkc, mkc, umr_en, 1);
386 MLX5_SET(mkc, mkc, lw, 1);
387 MLX5_SET(mkc, mkc, lr, 1);
cdbd0d2b 388 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
3608ae77
TT		 389
390 MLX5_SET(mkc, mkc, qpn, 0xffffff);
391 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
ec8b9981 392 MLX5_SET64(mkc, mkc, len, npages << page_shift);
3608ae77
TT		 393 MLX5_SET(mkc, mkc, translations_octword_size,
394 MLX5_MTT_OCTW(npages));
ec8b9981 395 MLX5_SET(mkc, mkc, log_page_size, page_shift);
3608ae77 396
ec8b9981 397 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
3608ae77
TT		 398
399 kvfree(in);
400 return err;
401}
402
a43b25da 403static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
ec8b9981 404{
422d4c40 405 u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq));
ec8b9981 406
a43b25da 407 return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey);
ec8b9981
TT		 408}
409
b8a98a4c
TT		 410static inline u64 mlx5e_get_mpwqe_offset(struct mlx5e_rq *rq, u16 wqe_ix)
411{
412 return (wqe_ix << MLX5E_LOG_ALIGNED_MPWQE_PPW) << PAGE_SHIFT;
413}
414
069d1146
TT		 415static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
416{
417 struct mlx5e_wqe_frag_info next_frag, *prev;
418 int i;
419
420 next_frag.di = &rq->wqe.di[0];
421 next_frag.offset = 0;
422 prev = NULL;
423
424 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
425 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
426 struct mlx5e_wqe_frag_info *frag =
427 &rq->wqe.frags[i << rq->wqe.info.log_num_frags];
428 int f;
429
430 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
431 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
432 next_frag.di++;
433 next_frag.offset = 0;
434 if (prev)
435 prev->last_in_page = true;
436 }
437 *frag = next_frag;
438
439 /* prepare next */
440 next_frag.offset += frag_info[f].frag_stride;
441 prev = frag;
442 }
443 }
444
445 if (prev)
446 prev->last_in_page = true;
447}
448
449static int mlx5e_init_di_list(struct mlx5e_rq *rq,
450 struct mlx5e_params *params,
451 int wq_sz, int cpu)
452{
453 int len = wq_sz << rq->wqe.info.log_num_frags;
454
84ca176b 455 rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)),
069d1146
TT		 456 GFP_KERNEL, cpu_to_node(cpu));
457 if (!rq->wqe.di)
458 return -ENOMEM;
459
460 mlx5e_init_frags_partition(rq);
461
462 return 0;
463}
464
465static void mlx5e_free_di_list(struct mlx5e_rq *rq)
466{
467 kvfree(rq->wqe.di);
468}
469
3b77235b 470static int mlx5e_alloc_rq(struct mlx5e_channel *c,
6a9764ef
SM		 471 struct mlx5e_params *params,
472 struct mlx5e_rq_param *rqp,
3b77235b 473 struct mlx5e_rq *rq)
f62b8bb8 474{
60bbf7ee 475 struct page_pool_params pp_params = { 0 };
a43b25da 476 struct mlx5_core_dev *mdev = c->mdev;
6a9764ef 477 void *rqc = rqp->rqc;
f62b8bb8 478 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
069d1146 479 u32 pool_size;
f62b8bb8
AV		 480 int wq_sz;
481 int err;
482 int i;
483
231243c8 484 rqp->wq.db_numa_node = cpu_to_node(c->cpu);
311c7c71 485
6a9764ef 486 rq->wq_type = params->rq_wq_type;
7e426671
TT		 487 rq->pdev = c->pdev;
488 rq->netdev = c->netdev;
a43b25da 489 rq->tstamp = c->tstamp;
7c39afb3 490 rq->clock = &mdev->clock;
7e426671
TT		 491 rq->channel = c;
492 rq->ix = c->ix;
a43b25da 493 rq->mdev = mdev;
472a1e44 494 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
05909bab 495 rq->stats = &c->priv->channel_stats[c->ix].rq;
97bc402d 496
6a9764ef 497 rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
97bc402d
DB		 498 if (IS_ERR(rq->xdp_prog)) {
499 err = PTR_ERR(rq->xdp_prog);
500 rq->xdp_prog = NULL;
501 goto err_rq_wq_destroy;
502 }
7e426671 503
e213f5b6
WY		 504 err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix);
505 if (err < 0)
0ddf5432
JDB		 506 goto err_rq_wq_destroy;
507
bce2b2bf 508 rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
619a8f2a 509 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params);
60bbf7ee 510 pool_size = 1 << params->log_rq_mtu_frames;
b5503b99 511
6a9764ef 512 switch (rq->wq_type) {
461017cb 513 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
422d4c40
TT		 514 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
515 &rq->wq_ctrl);
516 if (err)
517 return err;
518
519 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
520
521 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
60bbf7ee
JDB		 522
523 pool_size = MLX5_MPWRQ_PAGES_PER_WQE << mlx5e_mpwqe_get_log_rq_size(params);
422d4c40 524
7cc6d77b 525 rq->post_wqes = mlx5e_post_rx_mpwqes;
6cd392a0 526 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
461017cb 527
20fd0c19 528 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe;
899a59d3
IT		 529#ifdef CONFIG_MLX5_EN_IPSEC
530 if (MLX5_IPSEC_DEV(mdev)) {
531 err = -EINVAL;
532 netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n");
533 goto err_rq_wq_destroy;
534 }
535#endif
20fd0c19
SM		 536 if (!rq->handle_rx_cqe) {
537 err = -EINVAL;
538 netdev_err(c->netdev, "RX handler of MPWQE RQ is not set, err %d\n", err);
539 goto err_rq_wq_destroy;
540 }
541
619a8f2a
TT		 542 rq->mpwqe.skb_from_cqe_mpwrq =
543 mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ?
544 mlx5e_skb_from_cqe_mpwrq_linear :
545 mlx5e_skb_from_cqe_mpwrq_nonlinear;
f1e4fc9b
TT		 546 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params);
547 rq->mpwqe.num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params));
1bfecfca 548
a43b25da 549 err = mlx5e_create_rq_umr_mkey(mdev, rq);
7e426671
TT		 550 if (err)
551 goto err_rq_wq_destroy;
ec8b9981
TT		 552 rq->mkey_be = cpu_to_be32(rq->umr_mkey.key);
553
554 err = mlx5e_rq_alloc_mpwqe_info(rq, c);
555 if (err)
069d1146 556 goto err_free;
461017cb 557 break;
99cbfa93
TT		 558 default: /* MLX5_WQ_TYPE_CYCLIC */
559 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
560 &rq->wq_ctrl);
422d4c40
TT		 561 if (err)
562 return err;
563
564 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR];
565
99cbfa93 566 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
422d4c40 567
069d1146
TT		 568 rq->wqe.info = rqp->frags_info;
569 rq->wqe.frags =
84ca176b
KC		 570 kvzalloc_node(array_size(sizeof(*rq->wqe.frags),
571 (wq_sz << rq->wqe.info.log_num_frags)),
069d1146 572 GFP_KERNEL, cpu_to_node(c->cpu));
47a6ca3f
WY		 573 if (!rq->wqe.frags) {
574 err = -ENOMEM;
069d1146 575 goto err_free;
47a6ca3f 576 }
069d1146
TT		 577
578 err = mlx5e_init_di_list(rq, params, wq_sz, c->cpu);
579 if (err)
580 goto err_free;
7cc6d77b 581 rq->post_wqes = mlx5e_post_rx_wqes;
6cd392a0 582 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
461017cb 583
899a59d3
IT		 584#ifdef CONFIG_MLX5_EN_IPSEC
585 if (c->priv->ipsec)
586 rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
587 else
588#endif
589 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe;
20fd0c19 590 if (!rq->handle_rx_cqe) {
20fd0c19
SM		 591 err = -EINVAL;
592 netdev_err(c->netdev, "RX handler of RQ is not set, err %d\n", err);
069d1146 593 goto err_free;
20fd0c19
SM		 594 }
595
069d1146
TT		 596 rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(mdev, params) ?
597 mlx5e_skb_from_cqe_linear :
598 mlx5e_skb_from_cqe_nonlinear;
7e426671 599 rq->mkey_be = c->mkey_be;
461017cb 600 }
f62b8bb8 601
60bbf7ee 602 /* Create a page_pool and register it with rxq */
069d1146 603 pp_params.order = 0;
60bbf7ee
JDB		 604 pp_params.flags = 0; /* No-internal DMA mapping in page_pool */
605 pp_params.pool_size = pool_size;
606 pp_params.nid = cpu_to_node(c->cpu);
607 pp_params.dev = c->pdev;
608 pp_params.dma_dir = rq->buff.map_dir;
609
610 /* page_pool can be used even when there is no rq->xdp_prog,
611 * given page_pool does not handle DMA mapping there is no
612 * required state to clear. And page_pool gracefully handle
613 * elevated refcnt.
614 */
615 rq->page_pool = page_pool_create(&pp_params);
616 if (IS_ERR(rq->page_pool)) {
60bbf7ee
JDB		 617 err = PTR_ERR(rq->page_pool);
618 rq->page_pool = NULL;
069d1146 619 goto err_free;
84f5e3fb 620 }
60bbf7ee
JDB		 621 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
622 MEM_TYPE_PAGE_POOL, rq->page_pool);
623 if (err)
069d1146 624 goto err_free;
84f5e3fb 625
f62b8bb8 626 for (i = 0; i < wq_sz; i++) {
4c2af5cc 627 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
99cbfa93 628 struct mlx5e_rx_wqe_ll *wqe =
422d4c40 629 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
069d1146
TT		 630 u32 byte_count =
631 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
b8a98a4c 632 u64 dma_offset = mlx5e_get_mpwqe_offset(rq, i);
4c2af5cc 633
99cbfa93
TT		 634 wqe->data[0].addr = cpu_to_be64(dma_offset + rq->buff.headroom);
635 wqe->data[0].byte_count = cpu_to_be32(byte_count);
636 wqe->data[0].lkey = rq->mkey_be;
422d4c40 637 } else {
99cbfa93
TT		 638 struct mlx5e_rx_wqe_cyc *wqe =
639 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
069d1146
TT		 640 int f;
641
642 for (f = 0; f < rq->wqe.info.num_frags; f++) {
643 u32 frag_size = rq->wqe.info.arr[f].frag_size |
644 MLX5_HW_START_PADDING;
645
646 wqe->data[f].byte_count = cpu_to_be32(frag_size);
647 wqe->data[f].lkey = rq->mkey_be;
648 }
649 /* check if num_frags is not a pow of two */
650 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
651 wqe->data[f].byte_count = 0;
652 wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
653 wqe->data[f].addr = 0;
654 }
422d4c40 655 }
f62b8bb8
AV		 656 }
657
9a317425
AG		 658 INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work);
659
660 switch (params->rx_cq_moderation.cq_period_mode) {
661 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
662 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
663 break;
664 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
665 default:
666 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
667 }
668
4415a031
TT		 669 rq->page_cache.head = 0;
670 rq->page_cache.tail = 0;
671
f62b8bb8
AV		 672 return 0;
673
069d1146
TT		 674err_free:
675 switch (rq->wq_type) {
676 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
ca11b798 677 kvfree(rq->mpwqe.info);
069d1146
TT		 678 mlx5_core_destroy_mkey(mdev, &rq->umr_mkey);
679 break;
680 default: /* MLX5_WQ_TYPE_CYCLIC */
681 kvfree(rq->wqe.frags);
682 mlx5e_free_di_list(rq);
683 }
ec8b9981 684
f62b8bb8 685err_rq_wq_destroy:
97bc402d
DB		 686 if (rq->xdp_prog)
687 bpf_prog_put(rq->xdp_prog);
0ddf5432 688 xdp_rxq_info_unreg(&rq->xdp_rxq);
60bbf7ee
JDB		 689 if (rq->page_pool)
690 page_pool_destroy(rq->page_pool);
f62b8bb8
AV		 691 mlx5_wq_destroy(&rq->wq_ctrl);
692
693 return err;
694}
695
3b77235b 696static void mlx5e_free_rq(struct mlx5e_rq *rq)
f62b8bb8 697{
4415a031
TT		 698 int i;
699
86994156
RS		 700 if (rq->xdp_prog)
701 bpf_prog_put(rq->xdp_prog);
702
0ddf5432 703 xdp_rxq_info_unreg(&rq->xdp_rxq);
60bbf7ee
JDB		 704 if (rq->page_pool)
705 page_pool_destroy(rq->page_pool);
0ddf5432 706
461017cb
TT		 707 switch (rq->wq_type) {
708 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
ca11b798 709 kvfree(rq->mpwqe.info);
a43b25da 710 mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey);
461017cb 711 break;
99cbfa93 712 default: /* MLX5_WQ_TYPE_CYCLIC */
069d1146
TT		 713 kvfree(rq->wqe.frags);
714 mlx5e_free_di_list(rq);
461017cb
TT		 715 }
716
4415a031
TT		 717 for (i = rq->page_cache.head; i != rq->page_cache.tail;
718 i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
719 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
720
721 mlx5e_page_release(rq, dma_info, false);
722 }
f62b8bb8
AV		 723 mlx5_wq_destroy(&rq->wq_ctrl);
724}
725
6a9764ef
SM		 726static int mlx5e_create_rq(struct mlx5e_rq *rq,
727 struct mlx5e_rq_param *param)
f62b8bb8 728{
a43b25da 729 struct mlx5_core_dev *mdev = rq->mdev;
f62b8bb8
AV		 730
731 void *in;
732 void *rqc;
733 void *wq;
734 int inlen;
735 int err;
736
737 inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
738 sizeof(u64) * rq->wq_ctrl.buf.npages;
1b9a07ee 739 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 740 if (!in)
741 return -ENOMEM;
742
743 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
744 wq = MLX5_ADDR_OF(rqc, rqc, wq);
745
746 memcpy(rqc, param->rqc, sizeof(param->rqc));
747
97de9f31 748 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn);
f62b8bb8 749 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
f62b8bb8 750 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
68cdf5d6 751 MLX5_ADAPTER_PAGE_SHIFT);
f62b8bb8
AV		 752 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
753
3a2f7033
TT		 754 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
755 (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
f62b8bb8 756
7db22ffb 757 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
f62b8bb8
AV		 758
759 kvfree(in);
760
761 return err;
762}
763
36350114
GP		 764static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
765 int next_state)
f62b8bb8 766{
7cbaf9a3 767 struct mlx5_core_dev *mdev = rq->mdev;
f62b8bb8
AV		 768
769 void *in;
770 void *rqc;
771 int inlen;
772 int err;
773
774 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 775 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 776 if (!in)
777 return -ENOMEM;
778
779 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
780
781 MLX5_SET(modify_rq_in, in, rq_state, curr_state);
782 MLX5_SET(rqc, rqc, state, next_state);
783
7db22ffb 784 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
f62b8bb8
AV		 785
786 kvfree(in);
787
788 return err;
789}
790
102722fc
GE		 791static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable)
792{
793 struct mlx5e_channel *c = rq->channel;
794 struct mlx5e_priv *priv = c->priv;
795 struct mlx5_core_dev *mdev = priv->mdev;
796
797 void *in;
798 void *rqc;
799 int inlen;
800 int err;
801
802 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 803 in = kvzalloc(inlen, GFP_KERNEL);
102722fc
GE		 804 if (!in)
805 return -ENOMEM;
806
807 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
808
809 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
810 MLX5_SET64(modify_rq_in, in, modify_bitmask,
811 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS);
812 MLX5_SET(rqc, rqc, scatter_fcs, enable);
813 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
814
815 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
816
817 kvfree(in);
818
819 return err;
820}
821
36350114
GP		 822static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
823{
824 struct mlx5e_channel *c = rq->channel;
a43b25da 825 struct mlx5_core_dev *mdev = c->mdev;
36350114
GP		 826 void *in;
827 void *rqc;
828 int inlen;
829 int err;
830
831 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 832 in = kvzalloc(inlen, GFP_KERNEL);
36350114
GP		 833 if (!in)
834 return -ENOMEM;
835
836 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
837
838 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
83b502a1
AV		 839 MLX5_SET64(modify_rq_in, in, modify_bitmask,
840 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
36350114
GP		 841 MLX5_SET(rqc, rqc, vsd, vsd);
842 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
843
844 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
845
846 kvfree(in);
847
848 return err;
849}
850
3b77235b 851static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
f62b8bb8 852{
a43b25da 853 mlx5_core_destroy_rq(rq->mdev, rq->rqn);
f62b8bb8
AV		 854}
855
1e7477ae 856static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
f62b8bb8 857{
1e7477ae 858 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
f62b8bb8 859 struct mlx5e_channel *c = rq->channel;
a43b25da 860
422d4c40 861 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq));
f62b8bb8 862
1e7477ae 863 do {
422d4c40 864 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes)
f62b8bb8
AV		 865 return 0;
866
867 msleep(20);
1e7477ae
EBE		 868 } while (time_before(jiffies, exp_time));
869
870 netdev_warn(c->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
422d4c40 871 c->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes);
f62b8bb8
AV		 872
873 return -ETIMEDOUT;
874}
875
f2fde18c
SM		 876static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
877{
f2fde18c
SM		 878 __be16 wqe_ix_be;
879 u16 wqe_ix;
880
422d4c40
TT		 881 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
882 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
883
99cbfa93 884 /* UMR WQE (if in progress) is always at wq->head */
422d4c40 885 if (rq->mpwqe.umr_in_progress)
afab995e 886 rq->dealloc_wqe(rq, wq->head);
422d4c40
TT		 887
888 while (!mlx5_wq_ll_is_empty(wq)) {
99cbfa93 889 struct mlx5e_rx_wqe_ll *wqe;
422d4c40
TT		 890
891 wqe_ix_be = *wq->tail_next;
892 wqe_ix = be16_to_cpu(wqe_ix_be);
893 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix);
894 rq->dealloc_wqe(rq, wqe_ix);
895 mlx5_wq_ll_pop(wq, wqe_ix_be,
896 &wqe->next.next_wqe_index);
897 }
898 } else {
99cbfa93 899 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
422d4c40 900
99cbfa93
TT		 901 while (!mlx5_wq_cyc_is_empty(wq)) {
902 wqe_ix = mlx5_wq_cyc_get_tail(wq);
422d4c40 903 rq->dealloc_wqe(rq, wqe_ix);
99cbfa93 904 mlx5_wq_cyc_pop(wq);
422d4c40 905 }
accd5883 906 }
069d1146 907
f2fde18c
SM		 908}
909
f62b8bb8 910static int mlx5e_open_rq(struct mlx5e_channel *c,
6a9764ef 911 struct mlx5e_params *params,
f62b8bb8
AV		 912 struct mlx5e_rq_param *param,
913 struct mlx5e_rq *rq)
914{
915 int err;
916
6a9764ef 917 err = mlx5e_alloc_rq(c, params, param, rq);
f62b8bb8
AV		 918 if (err)
919 return err;
920
3b77235b 921 err = mlx5e_create_rq(rq, param);
f62b8bb8 922 if (err)
3b77235b 923 goto err_free_rq;
f62b8bb8 924
36350114 925 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
f62b8bb8 926 if (err)
3b77235b 927 goto err_destroy_rq;
f62b8bb8 928
9a317425 929 if (params->rx_dim_enabled)
af5a6c93 930 __set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
cb3c7fd4 931
f62b8bb8
AV		 932 return 0;
933
f62b8bb8
AV		 934err_destroy_rq:
935 mlx5e_destroy_rq(rq);
3b77235b
SM		 936err_free_rq:
937 mlx5e_free_rq(rq);
f62b8bb8
AV		 938
939 return err;
940}
941
acc6c595
SM		 942static void mlx5e_activate_rq(struct mlx5e_rq *rq)
943{
944 struct mlx5e_icosq *sq = &rq->channel->icosq;
ddf385e3 945 struct mlx5_wq_cyc *wq = &sq->wq;
acc6c595
SM		 946 struct mlx5e_tx_wqe *nopwqe;
947
ddf385e3
TT		 948 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
949
acc6c595
SM		 950 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
951 sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
ddf385e3
TT		 952 nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
953 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
acc6c595
SM		 954}
955
956static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
f62b8bb8 957{
c0f1147d 958 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
f62b8bb8 959 napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
acc6c595 960}
cb3c7fd4 961
acc6c595
SM		 962static void mlx5e_close_rq(struct mlx5e_rq *rq)
963{
9a317425 964 cancel_work_sync(&rq->dim.work);
f62b8bb8 965 mlx5e_destroy_rq(rq);
3b77235b
SM		 966 mlx5e_free_rx_descs(rq);
967 mlx5e_free_rq(rq);
f62b8bb8
AV		 968}
969
31391048 970static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
b5503b99 971{
ca11b798 972 kvfree(sq->db.di);
b5503b99
SM		 973}
974
31391048 975static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
b5503b99
SM		 976{
977 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
978
eec4edc9 979 sq->db.di = kvzalloc_node(array_size(wq_sz, sizeof(*sq->db.di)),
ca11b798 980 GFP_KERNEL, numa);
31391048
SM		 981 if (!sq->db.di) {
982 mlx5e_free_xdpsq_db(sq);
b5503b99
SM		 983 return -ENOMEM;
984 }
985
986 return 0;
987}
988
31391048 989static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
6a9764ef 990 struct mlx5e_params *params,
31391048
SM		 991 struct mlx5e_sq_param *param,
992 struct mlx5e_xdpsq *sq)
993{
994 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
a43b25da 995 struct mlx5_core_dev *mdev = c->mdev;
ddf385e3 996 struct mlx5_wq_cyc *wq = &sq->wq;
31391048
SM		 997 int err;
998
999 sq->pdev = c->pdev;
1000 sq->mkey_be = c->mkey_be;
1001 sq->channel = c;
1002 sq->uar_map = mdev->mlx5e_res.bfreg.map;
6a9764ef 1003 sq->min_inline_mode = params->tx_min_inline_mode;
31391048 1004
231243c8 1005 param->wq.db_numa_node = cpu_to_node(c->cpu);
ddf385e3 1006 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
31391048
SM		 1007 if (err)
1008 return err;
ddf385e3 1009 wq->db = &wq->db[MLX5_SND_DBR];
31391048 1010
231243c8 1011 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
31391048
SM		 1012 if (err)
1013 goto err_sq_wq_destroy;
1014
1015 return 0;
1016
1017err_sq_wq_destroy:
1018 mlx5_wq_destroy(&sq->wq_ctrl);
1019
1020 return err;
1021}
1022
1023static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
1024{
1025 mlx5e_free_xdpsq_db(sq);
1026 mlx5_wq_destroy(&sq->wq_ctrl);
1027}
1028
1029static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
f62b8bb8 1030{
ca11b798 1031 kvfree(sq->db.ico_wqe);
f62b8bb8
AV		 1032}
1033
31391048 1034static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
f10b7cc7
SM		 1035{
1036 u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1037
eec4edc9
KC		 1038 sq->db.ico_wqe = kvzalloc_node(array_size(wq_sz,
1039 sizeof(*sq->db.ico_wqe)),
ca11b798 1040 GFP_KERNEL, numa);
f10b7cc7
SM		 1041 if (!sq->db.ico_wqe)
1042 return -ENOMEM;
1043
1044 return 0;
1045}
1046
31391048 1047static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
31391048
SM		 1048 struct mlx5e_sq_param *param,
1049 struct mlx5e_icosq *sq)
f10b7cc7 1050{
31391048 1051 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
a43b25da 1052 struct mlx5_core_dev *mdev = c->mdev;
ddf385e3 1053 struct mlx5_wq_cyc *wq = &sq->wq;
31391048 1054 int err;
f10b7cc7 1055
31391048
SM		 1056 sq->channel = c;
1057 sq->uar_map = mdev->mlx5e_res.bfreg.map;
f62b8bb8 1058
231243c8 1059 param->wq.db_numa_node = cpu_to_node(c->cpu);
ddf385e3 1060 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
31391048
SM		 1061 if (err)
1062 return err;
ddf385e3 1063 wq->db = &wq->db[MLX5_SND_DBR];
f62b8bb8 1064
231243c8 1065 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
31391048
SM		 1066 if (err)
1067 goto err_sq_wq_destroy;
1068
f62b8bb8 1069 return 0;
31391048
SM		 1070
1071err_sq_wq_destroy:
1072 mlx5_wq_destroy(&sq->wq_ctrl);
1073
1074 return err;
f62b8bb8
AV		 1075}
1076
31391048 1077static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
f10b7cc7 1078{
31391048
SM		 1079 mlx5e_free_icosq_db(sq);
1080 mlx5_wq_destroy(&sq->wq_ctrl);
f10b7cc7
SM		 1081}
1082
31391048 1083static void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
f10b7cc7 1084{
ca11b798
TT		 1085 kvfree(sq->db.wqe_info);
1086 kvfree(sq->db.dma_fifo);
f10b7cc7
SM		 1087}
1088
31391048 1089static int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
b5503b99 1090{
31391048
SM		 1091 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1092 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1093
eec4edc9
KC		 1094 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz,
1095 sizeof(*sq->db.dma_fifo)),
ca11b798 1096 GFP_KERNEL, numa);
eec4edc9
KC		 1097 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz,
1098 sizeof(*sq->db.wqe_info)),
ca11b798 1099 GFP_KERNEL, numa);
77bdf895 1100 if (!sq->db.dma_fifo || !sq->db.wqe_info) {
31391048
SM		 1101 mlx5e_free_txqsq_db(sq);
1102 return -ENOMEM;
b5503b99 1103 }
31391048
SM		 1104
1105 sq->dma_fifo_mask = df_sz - 1;
1106
1107 return 0;
b5503b99
SM		 1108}
1109
db75373c 1110static void mlx5e_sq_recover(struct work_struct *work);
31391048 1111static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
acc6c595 1112 int txq_ix,
6a9764ef 1113 struct mlx5e_params *params,
31391048 1114 struct mlx5e_sq_param *param,
05909bab
EBE		 1115 struct mlx5e_txqsq *sq,
1116 int tc)
f62b8bb8 1117{
31391048 1118 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
a43b25da 1119 struct mlx5_core_dev *mdev = c->mdev;
ddf385e3 1120 struct mlx5_wq_cyc *wq = &sq->wq;
f62b8bb8
AV		 1121 int err;
1122
f10b7cc7 1123 sq->pdev = c->pdev;
a43b25da 1124 sq->tstamp = c->tstamp;
7c39afb3 1125 sq->clock = &mdev->clock;
f10b7cc7
SM		 1126 sq->mkey_be = c->mkey_be;
1127 sq->channel = c;
acc6c595 1128 sq->txq_ix = txq_ix;
aff26157 1129 sq->uar_map = mdev->mlx5e_res.bfreg.map;
6a9764ef 1130 sq->min_inline_mode = params->tx_min_inline_mode;
05909bab 1131 sq->stats = &c->priv->channel_stats[c->ix].sq[tc];
db75373c 1132 INIT_WORK(&sq->recover.recover_work, mlx5e_sq_recover);
2ac9cfe7
IT		 1133 if (MLX5_IPSEC_DEV(c->priv->mdev))
1134 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
bf239741
IL		 1135 if (mlx5_accel_is_tls_device(c->priv->mdev))
1136 set_bit(MLX5E_SQ_STATE_TLS, &sq->state);
f10b7cc7 1137
231243c8 1138 param->wq.db_numa_node = cpu_to_node(c->cpu);
ddf385e3 1139 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
f62b8bb8 1140 if (err)
aff26157 1141 return err;
ddf385e3 1142 wq->db = &wq->db[MLX5_SND_DBR];
f62b8bb8 1143
231243c8 1144 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
7ec0bb22 1145 if (err)
f62b8bb8
AV		 1146 goto err_sq_wq_destroy;
1147
cbce4f44
TG		 1148 INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work);
1149 sq->dim.mode = params->tx_cq_moderation.cq_period_mode;
1150
f62b8bb8
AV		 1151 return 0;
1152
1153err_sq_wq_destroy:
1154 mlx5_wq_destroy(&sq->wq_ctrl);
1155
f62b8bb8
AV		 1156 return err;
1157}
1158
31391048 1159static void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
f62b8bb8 1160{
31391048 1161 mlx5e_free_txqsq_db(sq);
f62b8bb8 1162 mlx5_wq_destroy(&sq->wq_ctrl);
f62b8bb8
AV		 1163}
1164
33ad9711
SM		 1165struct mlx5e_create_sq_param {
1166 struct mlx5_wq_ctrl *wq_ctrl;
1167 u32 cqn;
1168 u32 tisn;
1169 u8 tis_lst_sz;
1170 u8 min_inline_mode;
1171};
1172
a43b25da 1173static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
33ad9711
SM		 1174 struct mlx5e_sq_param *param,
1175 struct mlx5e_create_sq_param *csp,
1176 u32 *sqn)
f62b8bb8 1177{
f62b8bb8
AV		 1178 void *in;
1179 void *sqc;
1180 void *wq;
1181 int inlen;
1182 int err;
1183
1184 inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
33ad9711 1185 sizeof(u64) * csp->wq_ctrl->buf.npages;
1b9a07ee 1186 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1187 if (!in)
1188 return -ENOMEM;
1189
1190 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1191 wq = MLX5_ADDR_OF(sqc, sqc, wq);
1192
1193 memcpy(sqc, param->sqc, sizeof(param->sqc));
33ad9711
SM		 1194 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz);
1195 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn);
1196 MLX5_SET(sqc, sqc, cqn, csp->cqn);
a6f402e4
SM		 1197
1198 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
33ad9711 1199 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode);
a6f402e4 1200
33ad9711 1201 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
db75373c 1202 MLX5_SET(sqc, sqc, flush_in_error_en, 1);
f62b8bb8
AV		 1203
1204 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
a43b25da 1205 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.bfreg.index);
33ad9711 1206 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift -
68cdf5d6 1207 MLX5_ADAPTER_PAGE_SHIFT);
33ad9711 1208 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma);
f62b8bb8 1209
3a2f7033
TT		 1210 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,
1211 (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
f62b8bb8 1212
33ad9711 1213 err = mlx5_core_create_sq(mdev, in, inlen, sqn);
f62b8bb8
AV		 1214
1215 kvfree(in);
1216
1217 return err;
1218}
1219
33ad9711
SM		 1220struct mlx5e_modify_sq_param {
1221 int curr_state;
1222 int next_state;
1223 bool rl_update;
1224 int rl_index;
1225};
1226
a43b25da 1227static int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
33ad9711 1228 struct mlx5e_modify_sq_param *p)
f62b8bb8 1229{
f62b8bb8
AV		 1230 void *in;
1231 void *sqc;
1232 int inlen;
1233 int err;
1234
1235 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1b9a07ee 1236 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1237 if (!in)
1238 return -ENOMEM;
1239
1240 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1241
33ad9711
SM		 1242 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state);
1243 MLX5_SET(sqc, sqc, state, p->next_state);
1244 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) {
507f0c81 1245 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
33ad9711 1246 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index);
507f0c81 1247 }
f62b8bb8 1248
33ad9711 1249 err = mlx5_core_modify_sq(mdev, sqn, in, inlen);
f62b8bb8
AV		 1250
1251 kvfree(in);
1252
1253 return err;
1254}
1255
a43b25da 1256static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
33ad9711 1257{
a43b25da 1258 mlx5_core_destroy_sq(mdev, sqn);
f62b8bb8
AV		 1259}
1260
a43b25da 1261static int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
31391048
SM		 1262 struct mlx5e_sq_param *param,
1263 struct mlx5e_create_sq_param *csp,
1264 u32 *sqn)
f62b8bb8 1265{
33ad9711 1266 struct mlx5e_modify_sq_param msp = {0};
31391048
SM		 1267 int err;
1268
a43b25da 1269 err = mlx5e_create_sq(mdev, param, csp, sqn);
31391048
SM		 1270 if (err)
1271 return err;
1272
1273 msp.curr_state = MLX5_SQC_STATE_RST;
1274 msp.next_state = MLX5_SQC_STATE_RDY;
a43b25da 1275 err = mlx5e_modify_sq(mdev, *sqn, &msp);
31391048 1276 if (err)
a43b25da 1277 mlx5e_destroy_sq(mdev, *sqn);
31391048
SM		 1278
1279 return err;
1280}
1281
7f859ecf
SM		 1282static int mlx5e_set_sq_maxrate(struct net_device *dev,
1283 struct mlx5e_txqsq *sq, u32 rate);
1284
31391048 1285static int mlx5e_open_txqsq(struct mlx5e_channel *c,
a43b25da 1286 u32 tisn,
acc6c595 1287 int txq_ix,
6a9764ef 1288 struct mlx5e_params *params,
31391048 1289 struct mlx5e_sq_param *param,
05909bab
EBE		 1290 struct mlx5e_txqsq *sq,
1291 int tc)
31391048
SM		 1292{
1293 struct mlx5e_create_sq_param csp = {};
7f859ecf 1294 u32 tx_rate;
f62b8bb8
AV		 1295 int err;
1296
05909bab 1297 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);
f62b8bb8
AV		 1298 if (err)
1299 return err;
1300
a43b25da 1301 csp.tisn = tisn;
31391048 1302 csp.tis_lst_sz = 1;
33ad9711
SM		 1303 csp.cqn = sq->cq.mcq.cqn;
1304 csp.wq_ctrl = &sq->wq_ctrl;
1305 csp.min_inline_mode = sq->min_inline_mode;
a43b25da 1306 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
f62b8bb8 1307 if (err)
31391048 1308 goto err_free_txqsq;
f62b8bb8 1309
a43b25da 1310 tx_rate = c->priv->tx_rates[sq->txq_ix];
7f859ecf 1311 if (tx_rate)
a43b25da 1312 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
7f859ecf 1313
cbce4f44
TG		 1314 if (params->tx_dim_enabled)
1315 sq->state |= BIT(MLX5E_SQ_STATE_AM);
1316
f62b8bb8
AV		 1317 return 0;
1318
31391048 1319err_free_txqsq:
3b77235b 1320 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
31391048 1321 mlx5e_free_txqsq(sq);
f62b8bb8
AV		 1322
1323 return err;
1324}
1325
db75373c
EBE		 1326static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq)
1327{
1328 WARN_ONCE(sq->cc != sq->pc,
1329 "SQ 0x%x: cc (0x%x) != pc (0x%x)\n",
1330 sq->sqn, sq->cc, sq->pc);
1331 sq->cc = 0;
1332 sq->dma_fifo_cc = 0;
1333 sq->pc = 0;
1334}
1335
acc6c595
SM		 1336static void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
1337{
a43b25da 1338 sq->txq = netdev_get_tx_queue(sq->channel->netdev, sq->txq_ix);
db75373c 1339 clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state);
acc6c595
SM		 1340 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1341 netdev_tx_reset_queue(sq->txq);
1342 netif_tx_start_queue(sq->txq);
1343}
1344
f62b8bb8
AV		 1345static inline void netif_tx_disable_queue(struct netdev_queue *txq)
1346{
1347 __netif_tx_lock_bh(txq);
1348 netif_tx_stop_queue(txq);
1349 __netif_tx_unlock_bh(txq);
1350}
1351
acc6c595 1352static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
f62b8bb8 1353{
33ad9711 1354 struct mlx5e_channel *c = sq->channel;
ddf385e3 1355 struct mlx5_wq_cyc *wq = &sq->wq;
33ad9711 1356
c0f1147d 1357 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
6e8dd6d6 1358 /* prevent netif_tx_wake_queue */
33ad9711 1359 napi_synchronize(&c->napi);
29429f33 1360
31391048 1361 netif_tx_disable_queue(sq->txq);
f62b8bb8 1362
31391048 1363 /* last doorbell out, godspeed .. */
ddf385e3
TT		 1364 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
1365 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
31391048 1366 struct mlx5e_tx_wqe *nop;
864b2d71 1367
ddf385e3
TT		 1368 sq->db.wqe_info[pi].skb = NULL;
1369 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
1370 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
29429f33 1371 }
acc6c595
SM		 1372}
1373
1374static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
1375{
1376 struct mlx5e_channel *c = sq->channel;
a43b25da 1377 struct mlx5_core_dev *mdev = c->mdev;
05d3ac97 1378 struct mlx5_rate_limit rl = {0};
f62b8bb8 1379
a43b25da 1380 mlx5e_destroy_sq(mdev, sq->sqn);
05d3ac97
BW		 1381 if (sq->rate_limit) {
1382 rl.rate = sq->rate_limit;
1383 mlx5_rl_remove_rate(mdev, &rl);
1384 }
31391048
SM		 1385 mlx5e_free_txqsq_descs(sq);
1386 mlx5e_free_txqsq(sq);
1387}
1388
db75373c
EBE		 1389static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq)
1390{
1391 unsigned long exp_time = jiffies + msecs_to_jiffies(2000);
1392
1393 while (time_before(jiffies, exp_time)) {
1394 if (sq->cc == sq->pc)
1395 return 0;
1396
1397 msleep(20);
1398 }
1399
1400 netdev_err(sq->channel->netdev,
1401 "Wait for SQ 0x%x flush timeout (sq cc = 0x%x, sq pc = 0x%x)\n",
1402 sq->sqn, sq->cc, sq->pc);
1403
1404 return -ETIMEDOUT;
1405}
1406
1407static int mlx5e_sq_to_ready(struct mlx5e_txqsq *sq, int curr_state)
1408{
1409 struct mlx5_core_dev *mdev = sq->channel->mdev;
1410 struct net_device *dev = sq->channel->netdev;
1411 struct mlx5e_modify_sq_param msp = {0};
1412 int err;
1413
1414 msp.curr_state = curr_state;
1415 msp.next_state = MLX5_SQC_STATE_RST;
1416
1417 err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
1418 if (err) {
1419 netdev_err(dev, "Failed to move sq 0x%x to reset\n", sq->sqn);
1420 return err;
1421 }
1422
1423 memset(&msp, 0, sizeof(msp));
1424 msp.curr_state = MLX5_SQC_STATE_RST;
1425 msp.next_state = MLX5_SQC_STATE_RDY;
1426
1427 err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
1428 if (err) {
1429 netdev_err(dev, "Failed to move sq 0x%x to ready\n", sq->sqn);
1430 return err;
1431 }
1432
1433 return 0;
1434}
1435
1436static void mlx5e_sq_recover(struct work_struct *work)
1437{
1438 struct mlx5e_txqsq_recover *recover =
1439 container_of(work, struct mlx5e_txqsq_recover,
1440 recover_work);
1441 struct mlx5e_txqsq *sq = container_of(recover, struct mlx5e_txqsq,
1442 recover);
1443 struct mlx5_core_dev *mdev = sq->channel->mdev;
1444 struct net_device *dev = sq->channel->netdev;
1445 u8 state;
1446 int err;
1447
1448 err = mlx5_core_query_sq_state(mdev, sq->sqn, &state);
1449 if (err) {
1450 netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n",
1451 sq->sqn, err);
1452 return;
1453 }
1454
1455 if (state != MLX5_RQC_STATE_ERR) {
1456 netdev_err(dev, "SQ 0x%x not in ERROR state\n", sq->sqn);
1457 return;
1458 }
1459
1460 netif_tx_disable_queue(sq->txq);
1461
1462 if (mlx5e_wait_for_sq_flush(sq))
1463 return;
1464
1465 /* If the interval between two consecutive recovers per SQ is too
1466 * short, don't recover to avoid infinite loop of ERR_CQE -> recover.
1467 * If we reached this state, there is probably a bug that needs to be
1468 * fixed. let's keep the queue close and let tx timeout cleanup.
1469 */
1470 if (jiffies_to_msecs(jiffies - recover->last_recover) <
1471 MLX5E_SQ_RECOVER_MIN_INTERVAL) {
1472 netdev_err(dev, "Recover SQ 0x%x canceled, too many error CQEs\n",
1473 sq->sqn);
1474 return;
1475 }
1476
1477 /* At this point, no new packets will arrive from the stack as TXQ is
1478 * marked with QUEUE_STATE_DRV_XOFF. In addition, NAPI cleared all
1479 * pending WQEs. SQ can safely reset the SQ.
1480 */
1481 if (mlx5e_sq_to_ready(sq, state))
1482 return;
1483
1484 mlx5e_reset_txqsq_cc_pc(sq);
05909bab 1485 sq->stats->recover++;
db75373c
EBE		 1486 recover->last_recover = jiffies;
1487 mlx5e_activate_txqsq(sq);
1488}
1489
31391048 1490static int mlx5e_open_icosq(struct mlx5e_channel *c,
6a9764ef 1491 struct mlx5e_params *params,
31391048
SM		 1492 struct mlx5e_sq_param *param,
1493 struct mlx5e_icosq *sq)
1494{
1495 struct mlx5e_create_sq_param csp = {};
1496 int err;
1497
6a9764ef 1498 err = mlx5e_alloc_icosq(c, param, sq);
31391048
SM		 1499 if (err)
1500 return err;
1501
1502 csp.cqn = sq->cq.mcq.cqn;
1503 csp.wq_ctrl = &sq->wq_ctrl;
6a9764ef 1504 csp.min_inline_mode = params->tx_min_inline_mode;
31391048 1505 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
a43b25da 1506 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
31391048
SM		 1507 if (err)
1508 goto err_free_icosq;
1509
1510 return 0;
1511
1512err_free_icosq:
1513 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1514 mlx5e_free_icosq(sq);
1515
1516 return err;
1517}
1518
1519static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
1520{
1521 struct mlx5e_channel *c = sq->channel;
1522
1523 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1524 napi_synchronize(&c->napi);
1525
a43b25da 1526 mlx5e_destroy_sq(c->mdev, sq->sqn);
31391048
SM		 1527 mlx5e_free_icosq(sq);
1528}
1529
1530static int mlx5e_open_xdpsq(struct mlx5e_channel *c,
6a9764ef 1531 struct mlx5e_params *params,
31391048
SM		 1532 struct mlx5e_sq_param *param,
1533 struct mlx5e_xdpsq *sq)
1534{
1535 unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
1536 struct mlx5e_create_sq_param csp = {};
31391048
SM		 1537 unsigned int inline_hdr_sz = 0;
1538 int err;
1539 int i;
1540
6a9764ef 1541 err = mlx5e_alloc_xdpsq(c, params, param, sq);
31391048
SM		 1542 if (err)
1543 return err;
1544
1545 csp.tis_lst_sz = 1;
a43b25da 1546 csp.tisn = c->priv->tisn[0]; /* tc = 0 */
31391048
SM		 1547 csp.cqn = sq->cq.mcq.cqn;
1548 csp.wq_ctrl = &sq->wq_ctrl;
1549 csp.min_inline_mode = sq->min_inline_mode;
1550 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
a43b25da 1551 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
31391048
SM		 1552 if (err)
1553 goto err_free_xdpsq;
1554
1555 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
1556 inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
1557 ds_cnt++;
1558 }
1559
1560 /* Pre initialize fixed WQE fields */
1561 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) {
1562 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i);
1563 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
1564 struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
1565 struct mlx5_wqe_data_seg *dseg;
1566
1567 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
1568 eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
1569
1570 dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1);
1571 dseg->lkey = sq->mkey_be;
1572 }
1573
1574 return 0;
1575
1576err_free_xdpsq:
1577 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1578 mlx5e_free_xdpsq(sq);
1579
1580 return err;
1581}
1582
1583static void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
1584{
1585 struct mlx5e_channel *c = sq->channel;
1586
1587 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1588 napi_synchronize(&c->napi);
1589
a43b25da 1590 mlx5e_destroy_sq(c->mdev, sq->sqn);
31391048
SM		 1591 mlx5e_free_xdpsq_descs(sq);
1592 mlx5e_free_xdpsq(sq);
f62b8bb8
AV		 1593}
1594
95b6c6a5
EBE		 1595static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev,
1596 struct mlx5e_cq_param *param,
1597 struct mlx5e_cq *cq)
f62b8bb8 1598{
f62b8bb8
AV		 1599 struct mlx5_core_cq *mcq = &cq->mcq;
1600 int eqn_not_used;
0b6e26ce 1601 unsigned int irqn;
f62b8bb8
AV		 1602 int err;
1603 u32 i;
1604
f62b8bb8
AV		 1605 err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1606 &cq->wq_ctrl);
1607 if (err)
1608 return err;
1609
1610 mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1611
f62b8bb8
AV		 1612 mcq->cqe_sz = 64;
1613 mcq->set_ci_db = cq->wq_ctrl.db.db;
1614 mcq->arm_db = cq->wq_ctrl.db.db + 1;
1615 *mcq->set_ci_db = 0;
1616 *mcq->arm_db = 0;
1617 mcq->vector = param->eq_ix;
1618 mcq->comp = mlx5e_completion_event;
1619 mcq->event = mlx5e_cq_error_event;
1620 mcq->irqn = irqn;
f62b8bb8
AV		 1621
1622 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1623 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1624
1625 cqe->op_own = 0xf1;
1626 }
1627
a43b25da 1628 cq->mdev = mdev;
f62b8bb8
AV		 1629
1630 return 0;
1631}
1632
95b6c6a5
EBE		 1633static int mlx5e_alloc_cq(struct mlx5e_channel *c,
1634 struct mlx5e_cq_param *param,
1635 struct mlx5e_cq *cq)
1636{
1637 struct mlx5_core_dev *mdev = c->priv->mdev;
1638 int err;
1639
231243c8
SM		 1640 param->wq.buf_numa_node = cpu_to_node(c->cpu);
1641 param->wq.db_numa_node = cpu_to_node(c->cpu);
95b6c6a5
EBE		 1642 param->eq_ix = c->ix;
1643
1644 err = mlx5e_alloc_cq_common(mdev, param, cq);
1645
1646 cq->napi = &c->napi;
1647 cq->channel = c;
1648
1649 return err;
1650}
1651
3b77235b 1652static void mlx5e_free_cq(struct mlx5e_cq *cq)
f62b8bb8 1653{
3a2f7033 1654 mlx5_wq_destroy(&cq->wq_ctrl);
f62b8bb8
AV		 1655}
1656
3b77235b 1657static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
f62b8bb8 1658{
a43b25da 1659 struct mlx5_core_dev *mdev = cq->mdev;
f62b8bb8
AV		 1660 struct mlx5_core_cq *mcq = &cq->mcq;
1661
1662 void *in;
1663 void *cqc;
1664 int inlen;
0b6e26ce 1665 unsigned int irqn_not_used;
f62b8bb8
AV		 1666 int eqn;
1667 int err;
1668
1669 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
3a2f7033 1670 sizeof(u64) * cq->wq_ctrl.buf.npages;
1b9a07ee 1671 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1672 if (!in)
1673 return -ENOMEM;
1674
1675 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1676
1677 memcpy(cqc, param->cqc, sizeof(param->cqc));
1678
3a2f7033 1679 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
1c1b5228 1680 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
f62b8bb8
AV		 1681
1682 mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1683
9908aa29 1684 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode);
f62b8bb8 1685 MLX5_SET(cqc, cqc, c_eqn, eqn);
30aa60b3 1686 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
3a2f7033 1687 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
68cdf5d6 1688 MLX5_ADAPTER_PAGE_SHIFT);
f62b8bb8
AV		 1689 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
1690
1691 err = mlx5_core_create_cq(mdev, mcq, in, inlen);
1692
1693 kvfree(in);
1694
1695 if (err)
1696 return err;
1697
1698 mlx5e_cq_arm(cq);
1699
1700 return 0;
1701}
1702
3b77235b 1703static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
f62b8bb8 1704{
a43b25da 1705 mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
f62b8bb8
AV		 1706}
1707
1708static int mlx5e_open_cq(struct mlx5e_channel *c,
9a317425 1709 struct net_dim_cq_moder moder,
f62b8bb8 1710 struct mlx5e_cq_param *param,
6a9764ef 1711 struct mlx5e_cq *cq)
f62b8bb8 1712{
a43b25da 1713 struct mlx5_core_dev *mdev = c->mdev;
f62b8bb8 1714 int err;
f62b8bb8 1715
3b77235b 1716 err = mlx5e_alloc_cq(c, param, cq);
f62b8bb8
AV		 1717 if (err)
1718 return err;
1719
3b77235b 1720 err = mlx5e_create_cq(cq, param);
f62b8bb8 1721 if (err)
3b77235b 1722 goto err_free_cq;
f62b8bb8 1723
7524a5d8 1724 if (MLX5_CAP_GEN(mdev, cq_moderation))
6a9764ef 1725 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts);
f62b8bb8
AV		 1726 return 0;
1727
3b77235b
SM		 1728err_free_cq:
1729 mlx5e_free_cq(cq);
f62b8bb8
AV		 1730
1731 return err;
1732}
1733
1734static void mlx5e_close_cq(struct mlx5e_cq *cq)
1735{
f62b8bb8 1736 mlx5e_destroy_cq(cq);
3b77235b 1737 mlx5e_free_cq(cq);
f62b8bb8
AV		 1738}
1739
231243c8
SM		 1740static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
1741{
1742 return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
1743}
1744
f62b8bb8 1745static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
6a9764ef 1746 struct mlx5e_params *params,
f62b8bb8
AV		 1747 struct mlx5e_channel_param *cparam)
1748{
f62b8bb8
AV		 1749 int err;
1750 int tc;
1751
1752 for (tc = 0; tc < c->num_tc; tc++) {
6a9764ef
SM		 1753 err = mlx5e_open_cq(c, params->tx_cq_moderation,
1754 &cparam->tx_cq, &c->sq[tc].cq);
f62b8bb8
AV		 1755 if (err)
1756 goto err_close_tx_cqs;
f62b8bb8
AV		 1757 }
1758
1759 return 0;
1760
1761err_close_tx_cqs:
1762 for (tc--; tc >= 0; tc--)
1763 mlx5e_close_cq(&c->sq[tc].cq);
1764
1765 return err;
1766}
1767
1768static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1769{
1770 int tc;
1771
1772 for (tc = 0; tc < c->num_tc; tc++)
1773 mlx5e_close_cq(&c->sq[tc].cq);
1774}
1775
1776static int mlx5e_open_sqs(struct mlx5e_channel *c,
6a9764ef 1777 struct mlx5e_params *params,
f62b8bb8
AV		 1778 struct mlx5e_channel_param *cparam)
1779{
05909bab
EBE		 1780 struct mlx5e_priv *priv = c->priv;
1781 int err, tc, max_nch = priv->profile->max_nch(priv->mdev);
f62b8bb8 1782
6a9764ef 1783 for (tc = 0; tc < params->num_tc; tc++) {
05909bab 1784 int txq_ix = c->ix + tc * max_nch;
acc6c595 1785
a43b25da 1786 err = mlx5e_open_txqsq(c, c->priv->tisn[tc], txq_ix,
05909bab 1787 params, &cparam->sq, &c->sq[tc], tc);
f62b8bb8
AV		 1788 if (err)
1789 goto err_close_sqs;
1790 }
1791
1792 return 0;
1793
1794err_close_sqs:
1795 for (tc--; tc >= 0; tc--)
31391048 1796 mlx5e_close_txqsq(&c->sq[tc]);
f62b8bb8
AV		 1797
1798 return err;
1799}
1800
1801static void mlx5e_close_sqs(struct mlx5e_channel *c)
1802{
1803 int tc;
1804
1805 for (tc = 0; tc < c->num_tc; tc++)
31391048 1806 mlx5e_close_txqsq(&c->sq[tc]);
f62b8bb8
AV		 1807}
1808
507f0c81 1809static int mlx5e_set_sq_maxrate(struct net_device *dev,
31391048 1810 struct mlx5e_txqsq *sq, u32 rate)
507f0c81
YP		 1811{
1812 struct mlx5e_priv *priv = netdev_priv(dev);
1813 struct mlx5_core_dev *mdev = priv->mdev;
33ad9711 1814 struct mlx5e_modify_sq_param msp = {0};
05d3ac97 1815 struct mlx5_rate_limit rl = {0};
507f0c81
YP		 1816 u16 rl_index = 0;
1817 int err;
1818
1819 if (rate == sq->rate_limit)
1820 /* nothing to do */
1821 return 0;
1822
05d3ac97
BW		 1823 if (sq->rate_limit) {
1824 rl.rate = sq->rate_limit;
507f0c81 1825 /* remove current rl index to free space to next ones */
05d3ac97
BW		 1826 mlx5_rl_remove_rate(mdev, &rl);
1827 }
507f0c81
YP		 1828
1829 sq->rate_limit = 0;
1830
1831 if (rate) {
05d3ac97
BW		 1832 rl.rate = rate;
1833 err = mlx5_rl_add_rate(mdev, &rl_index, &rl);
507f0c81
YP		 1834 if (err) {
1835 netdev_err(dev, "Failed configuring rate %u: %d\n",
1836 rate, err);
1837 return err;
1838 }
1839 }
1840
33ad9711
SM		 1841 msp.curr_state = MLX5_SQC_STATE_RDY;
1842 msp.next_state = MLX5_SQC_STATE_RDY;
1843 msp.rl_index = rl_index;
1844 msp.rl_update = true;
a43b25da 1845 err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
507f0c81
YP		 1846 if (err) {
1847 netdev_err(dev, "Failed configuring rate %u: %d\n",
1848 rate, err);
1849 /* remove the rate from the table */
1850 if (rate)
05d3ac97 1851 mlx5_rl_remove_rate(mdev, &rl);
507f0c81
YP		 1852 return err;
1853 }
1854
1855 sq->rate_limit = rate;
1856 return 0;
1857}
1858
1859static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1860{
1861 struct mlx5e_priv *priv = netdev_priv(dev);
1862 struct mlx5_core_dev *mdev = priv->mdev;
acc6c595 1863 struct mlx5e_txqsq *sq = priv->txq2sq[index];
507f0c81
YP		 1864 int err = 0;
1865
1866 if (!mlx5_rl_is_supported(mdev)) {
1867 netdev_err(dev, "Rate limiting is not supported on this device\n");
1868 return -EINVAL;
1869 }
1870
1871 /* rate is given in Mb/sec, HW config is in Kb/sec */
1872 rate = rate << 10;
1873
1874 /* Check whether rate in valid range, 0 is always valid */
1875 if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1876 netdev_err(dev, "TX rate %u, is not in range\n", rate);
1877 return -ERANGE;
1878 }
1879
1880 mutex_lock(&priv->state_lock);
1881 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1882 err = mlx5e_set_sq_maxrate(dev, sq, rate);
1883 if (!err)
1884 priv->tx_rates[index] = rate;
1885 mutex_unlock(&priv->state_lock);
1886
1887 return err;
1888}
1889
f62b8bb8 1890static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
6a9764ef 1891 struct mlx5e_params *params,
f62b8bb8
AV		 1892 struct mlx5e_channel_param *cparam,
1893 struct mlx5e_channel **cp)
1894{
9a317425 1895 struct net_dim_cq_moder icocq_moder = {0, 0};
f62b8bb8 1896 struct net_device *netdev = priv->netdev;
231243c8 1897 int cpu = mlx5e_get_cpu(priv, ix);
f62b8bb8 1898 struct mlx5e_channel *c;
a8c2eb15 1899 unsigned int irq;
f62b8bb8 1900 int err;
a8c2eb15 1901 int eqn;
f62b8bb8 1902
ca11b798 1903 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
f62b8bb8
AV		 1904 if (!c)
1905 return -ENOMEM;
1906
1907 c->priv = priv;
a43b25da
SM		 1908 c->mdev = priv->mdev;
1909 c->tstamp = &priv->tstamp;
f62b8bb8 1910 c->ix = ix;
231243c8 1911 c->cpu = cpu;
f62b8bb8
AV		 1912 c->pdev = &priv->mdev->pdev->dev;
1913 c->netdev = priv->netdev;
b50d292b 1914 c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
6a9764ef
SM		 1915 c->num_tc = params->num_tc;
1916 c->xdp = !!params->xdp_prog;
05909bab 1917 c->stats = &priv->channel_stats[ix].ch;
cb3c7fd4 1918
a8c2eb15
TT		 1919 mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);
1920 c->irq_desc = irq_to_desc(irq);
1921
f62b8bb8
AV		 1922 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1923
6a9764ef 1924 err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
f62b8bb8
AV		 1925 if (err)
1926 goto err_napi_del;
1927
6a9764ef 1928 err = mlx5e_open_tx_cqs(c, params, cparam);
d3c9bc27
TT		 1929 if (err)
1930 goto err_close_icosq_cq;
1931
6a9764ef 1932 err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq);
f62b8bb8
AV		 1933 if (err)
1934 goto err_close_tx_cqs;
f62b8bb8 1935
d7a0ecab 1936 /* XDP SQ CQ params are same as normal TXQ sq CQ params */
6a9764ef
SM		 1937 err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,
1938 &cparam->tx_cq, &c->rq.xdpsq.cq) : 0;
d7a0ecab
SM		 1939 if (err)
1940 goto err_close_rx_cq;
1941
f62b8bb8
AV		 1942 napi_enable(&c->napi);
1943
6a9764ef 1944 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq);
f62b8bb8
AV		 1945 if (err)
1946 goto err_disable_napi;
1947
6a9764ef 1948 err = mlx5e_open_sqs(c, params, cparam);
d3c9bc27
TT		 1949 if (err)
1950 goto err_close_icosq;
1951
6a9764ef 1952 err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq) : 0;
d7a0ecab
SM		 1953 if (err)
1954 goto err_close_sqs;
b5503b99 1955
6a9764ef 1956 err = mlx5e_open_rq(c, params, &cparam->rq, &c->rq);
f62b8bb8 1957 if (err)
b5503b99 1958 goto err_close_xdp_sq;
f62b8bb8 1959
f62b8bb8
AV		 1960 *cp = c;
1961
1962 return 0;
b5503b99 1963err_close_xdp_sq:
d7a0ecab 1964 if (c->xdp)
31391048 1965 mlx5e_close_xdpsq(&c->rq.xdpsq);
f62b8bb8
AV		 1966
1967err_close_sqs:
1968 mlx5e_close_sqs(c);
1969
d3c9bc27 1970err_close_icosq:
31391048 1971 mlx5e_close_icosq(&c->icosq);
d3c9bc27 1972
f62b8bb8
AV		 1973err_disable_napi:
1974 napi_disable(&c->napi);
d7a0ecab 1975 if (c->xdp)
31871f87 1976 mlx5e_close_cq(&c->rq.xdpsq.cq);
d7a0ecab
SM		 1977
1978err_close_rx_cq:
f62b8bb8
AV		 1979 mlx5e_close_cq(&c->rq.cq);
1980
1981err_close_tx_cqs:
1982 mlx5e_close_tx_cqs(c);
1983
d3c9bc27
TT		 1984err_close_icosq_cq:
1985 mlx5e_close_cq(&c->icosq.cq);
1986
f62b8bb8
AV		 1987err_napi_del:
1988 netif_napi_del(&c->napi);
ca11b798 1989 kvfree(c);
f62b8bb8
AV		 1990
1991 return err;
1992}
1993
acc6c595
SM		 1994static void mlx5e_activate_channel(struct mlx5e_channel *c)
1995{
1996 int tc;
1997
1998 for (tc = 0; tc < c->num_tc; tc++)
1999 mlx5e_activate_txqsq(&c->sq[tc]);
2000 mlx5e_activate_rq(&c->rq);
231243c8 2001 netif_set_xps_queue(c->netdev, get_cpu_mask(c->cpu), c->ix);
acc6c595
SM		 2002}
2003
2004static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
2005{
2006 int tc;
2007
2008 mlx5e_deactivate_rq(&c->rq);
2009 for (tc = 0; tc < c->num_tc; tc++)
2010 mlx5e_deactivate_txqsq(&c->sq[tc]);
2011}
2012
f62b8bb8
AV		 2013static void mlx5e_close_channel(struct mlx5e_channel *c)
2014{
2015 mlx5e_close_rq(&c->rq);
b5503b99 2016 if (c->xdp)
31391048 2017 mlx5e_close_xdpsq(&c->rq.xdpsq);
f62b8bb8 2018 mlx5e_close_sqs(c);
31391048 2019 mlx5e_close_icosq(&c->icosq);
f62b8bb8 2020 napi_disable(&c->napi);
b5503b99 2021 if (c->xdp)
31871f87 2022 mlx5e_close_cq(&c->rq.xdpsq.cq);
f62b8bb8
AV		 2023 mlx5e_close_cq(&c->rq.cq);
2024 mlx5e_close_tx_cqs(c);
d3c9bc27 2025 mlx5e_close_cq(&c->icosq.cq);
f62b8bb8 2026 netif_napi_del(&c->napi);
7ae92ae5 2027
ca11b798 2028 kvfree(c);
f62b8bb8
AV		 2029}
2030
069d1146
TT		 2031#define DEFAULT_FRAG_SIZE (2048)
2032
2033static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
2034 struct mlx5e_params *params,
2035 struct mlx5e_rq_frags_info *info)
2036{
2037 u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);
2038 int frag_size_max = DEFAULT_FRAG_SIZE;
2039 u32 buf_size = 0;
2040 int i;
2041
2042#ifdef CONFIG_MLX5_EN_IPSEC
2043 if (MLX5_IPSEC_DEV(mdev))
2044 byte_count += MLX5E_METADATA_ETHER_LEN;
2045#endif
2046
2047 if (mlx5e_rx_is_linear_skb(mdev, params)) {
2048 int frag_stride;
2049
2050 frag_stride = mlx5e_rx_get_linear_frag_sz(params);
2051 frag_stride = roundup_pow_of_two(frag_stride);
2052
2053 info->arr[0].frag_size = byte_count;
2054 info->arr[0].frag_stride = frag_stride;
2055 info->num_frags = 1;
2056 info->wqe_bulk = PAGE_SIZE / frag_stride;
2057 goto out;
2058 }
2059
2060 if (byte_count > PAGE_SIZE +
2061 (MLX5E_MAX_RX_FRAGS - 1) * frag_size_max)
2062 frag_size_max = PAGE_SIZE;
2063
2064 i = 0;
2065 while (buf_size < byte_count) {
2066 int frag_size = byte_count - buf_size;
2067
2068 if (i < MLX5E_MAX_RX_FRAGS - 1)
2069 frag_size = min(frag_size, frag_size_max);
2070
2071 info->arr[i].frag_size = frag_size;
2072 info->arr[i].frag_stride = roundup_pow_of_two(frag_size);
2073
2074 buf_size += frag_size;
2075 i++;
2076 }
2077 info->num_frags = i;
2078 /* number of different wqes sharing a page */
2079 info->wqe_bulk = 1 + (info->num_frags % 2);
2080
2081out:
2082 info->wqe_bulk = max_t(u8, info->wqe_bulk, 8);
2083 info->log_num_frags = order_base_2(info->num_frags);
2084}
2085
99cbfa93
TT		 2086static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs)
2087{
2088 int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs;
2089
2090 switch (wq_type) {
2091 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2092 sz += sizeof(struct mlx5e_rx_wqe_ll);
2093 break;
2094 default: /* MLX5_WQ_TYPE_CYCLIC */
2095 sz += sizeof(struct mlx5e_rx_wqe_cyc);
2096 }
2097
2098 return order_base_2(sz);
2099}
2100
f62b8bb8 2101static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
6a9764ef 2102 struct mlx5e_params *params,
f62b8bb8
AV		 2103 struct mlx5e_rq_param *param)
2104{
f1e4fc9b 2105 struct mlx5_core_dev *mdev = priv->mdev;
f62b8bb8
AV		 2106 void *rqc = param->rqc;
2107 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
99cbfa93 2108 int ndsegs = 1;
f62b8bb8 2109
6a9764ef 2110 switch (params->rq_wq_type) {
461017cb 2111 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
f1e4fc9b 2112 MLX5_SET(wq, wq, log_wqe_num_of_strides,
619a8f2a
TT		 2113 mlx5e_mpwqe_get_log_num_strides(mdev, params) -
2114 MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
f1e4fc9b 2115 MLX5_SET(wq, wq, log_wqe_stride_size,
619a8f2a
TT		 2116 mlx5e_mpwqe_get_log_stride_size(mdev, params) -
2117 MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
73281b78 2118 MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params));
461017cb 2119 break;
99cbfa93 2120 default: /* MLX5_WQ_TYPE_CYCLIC */
73281b78 2121 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
069d1146
TT		 2122 mlx5e_build_rq_frags_info(mdev, params, &param->frags_info);
2123 ndsegs = param->frags_info.num_frags;
461017cb
TT		 2124 }
2125
99cbfa93 2126 MLX5_SET(wq, wq, wq_type, params->rq_wq_type);
f62b8bb8 2127 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
99cbfa93
TT		 2128 MLX5_SET(wq, wq, log_wq_stride,
2129 mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs));
f1e4fc9b 2130 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.pdn);
593cf338 2131 MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
6a9764ef 2132 MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable);
102722fc 2133 MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en);
f62b8bb8 2134
f1e4fc9b 2135 param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev);
f62b8bb8
AV		 2136}
2137
7cbaf9a3 2138static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv,
2f0db879 2139 struct mlx5e_rq_param *param)
556dd1b9 2140{
7cbaf9a3 2141 struct mlx5_core_dev *mdev = priv->mdev;
556dd1b9
TT		 2142 void *rqc = param->rqc;
2143 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2144
99cbfa93
TT		 2145 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
2146 MLX5_SET(wq, wq, log_wq_stride,
2147 mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1));
7cbaf9a3 2148 MLX5_SET(rqc, rqc, counter_set_id, priv->drop_rq_q_counter);
2f0db879
GP		 2149
2150 param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev);
556dd1b9
TT		 2151}
2152
d3c9bc27
TT		 2153static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
2154 struct mlx5e_sq_param *param)
f62b8bb8
AV		 2155{
2156 void *sqc = param->sqc;
2157 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2158
f62b8bb8 2159 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
b50d292b 2160 MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn);
f62b8bb8 2161
311c7c71 2162 param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
d3c9bc27
TT		 2163}
2164
2165static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
6a9764ef 2166 struct mlx5e_params *params,
d3c9bc27
TT		 2167 struct mlx5e_sq_param *param)
2168{
2169 void *sqc = param->sqc;
2170 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2171
2172 mlx5e_build_sq_param_common(priv, param);
6a9764ef 2173 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
2ac9cfe7 2174 MLX5_SET(sqc, sqc, allow_swp, !!MLX5_IPSEC_DEV(priv->mdev));
f62b8bb8
AV		 2175}
2176
2177static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2178 struct mlx5e_cq_param *param)
2179{
2180 void *cqc = param->cqc;
2181
30aa60b3 2182 MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
f62b8bb8
AV		 2183}
2184
2185static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
6a9764ef 2186 struct mlx5e_params *params,
f62b8bb8
AV		 2187 struct mlx5e_cq_param *param)
2188{
73281b78 2189 struct mlx5_core_dev *mdev = priv->mdev;
f62b8bb8 2190 void *cqc = param->cqc;
461017cb 2191 u8 log_cq_size;
f62b8bb8 2192
6a9764ef 2193 switch (params->rq_wq_type) {
461017cb 2194 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
73281b78
TT		 2195 log_cq_size = mlx5e_mpwqe_get_log_rq_size(params) +
2196 mlx5e_mpwqe_get_log_num_strides(mdev, params);
461017cb 2197 break;
99cbfa93 2198 default: /* MLX5_WQ_TYPE_CYCLIC */
73281b78 2199 log_cq_size = params->log_rq_mtu_frames;
461017cb
TT		 2200 }
2201
2202 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
6a9764ef 2203 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
7219ab34
TT		 2204 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
2205 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
2206 }
f62b8bb8
AV		 2207
2208 mlx5e_build_common_cq_param(priv, param);
0088cbbc 2209 param->cq_period_mode = params->rx_cq_moderation.cq_period_mode;
f62b8bb8
AV		 2210}
2211
2212static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
6a9764ef 2213 struct mlx5e_params *params,
f62b8bb8
AV		 2214 struct mlx5e_cq_param *param)
2215{
2216 void *cqc = param->cqc;
2217
6a9764ef 2218 MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size);
f62b8bb8
AV		 2219
2220 mlx5e_build_common_cq_param(priv, param);
0088cbbc 2221 param->cq_period_mode = params->tx_cq_moderation.cq_period_mode;
f62b8bb8
AV		 2222}
2223
d3c9bc27 2224static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
6a9764ef
SM		 2225 u8 log_wq_size,
2226 struct mlx5e_cq_param *param)
d3c9bc27
TT		 2227{
2228 void *cqc = param->cqc;
2229
2230 MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
2231
2232 mlx5e_build_common_cq_param(priv, param);
9908aa29 2233
9a317425 2234 param->cq_period_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
d3c9bc27
TT		 2235}
2236
2237static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
6a9764ef
SM		 2238 u8 log_wq_size,
2239 struct mlx5e_sq_param *param)
d3c9bc27
TT		 2240{
2241 void *sqc = param->sqc;
2242 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2243
2244 mlx5e_build_sq_param_common(priv, param);
2245
2246 MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
bc77b240 2247 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
d3c9bc27
TT		 2248}
2249
b5503b99 2250static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
6a9764ef 2251 struct mlx5e_params *params,
b5503b99
SM		 2252 struct mlx5e_sq_param *param)
2253{
2254 void *sqc = param->sqc;
2255 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2256
2257 mlx5e_build_sq_param_common(priv, param);
6a9764ef 2258 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
b5503b99
SM		 2259}
2260
6a9764ef
SM		 2261static void mlx5e_build_channel_param(struct mlx5e_priv *priv,
2262 struct mlx5e_params *params,
2263 struct mlx5e_channel_param *cparam)
f62b8bb8 2264{
bc77b240 2265 u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
d3c9bc27 2266
6a9764ef
SM		 2267 mlx5e_build_rq_param(priv, params, &cparam->rq);
2268 mlx5e_build_sq_param(priv, params, &cparam->sq);
2269 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
2270 mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);
2271 mlx5e_build_rx_cq_param(priv, params, &cparam->rx_cq);
2272 mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq);
2273 mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq);
f62b8bb8
AV		 2274}
2275
55c2503d
SM		 2276int mlx5e_open_channels(struct mlx5e_priv *priv,
2277 struct mlx5e_channels *chs)
f62b8bb8 2278{
6b87663f 2279 struct mlx5e_channel_param *cparam;
03289b88 2280 int err = -ENOMEM;
f62b8bb8 2281 int i;
f62b8bb8 2282
6a9764ef 2283 chs->num = chs->params.num_channels;
03289b88 2284
ff9c852f 2285 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
ca11b798 2286 cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
acc6c595
SM		 2287 if (!chs->c || !cparam)
2288 goto err_free;
f62b8bb8 2289
6a9764ef 2290 mlx5e_build_channel_param(priv, &chs->params, cparam);
ff9c852f 2291 for (i = 0; i < chs->num; i++) {
6a9764ef 2292 err = mlx5e_open_channel(priv, i, &chs->params, cparam, &chs->c[i]);
f62b8bb8
AV		 2293 if (err)
2294 goto err_close_channels;
2295 }
2296
ca11b798 2297 kvfree(cparam);
f62b8bb8
AV		 2298 return 0;
2299
2300err_close_channels:
2301 for (i--; i >= 0; i--)
ff9c852f 2302 mlx5e_close_channel(chs->c[i]);
f62b8bb8 2303
acc6c595 2304err_free:
ff9c852f 2305 kfree(chs->c);
ca11b798 2306 kvfree(cparam);
ff9c852f 2307 chs->num = 0;
f62b8bb8
AV		 2308 return err;
2309}
2310
acc6c595 2311static void mlx5e_activate_channels(struct mlx5e_channels *chs)
f62b8bb8
AV		 2312{
2313 int i;
2314
acc6c595
SM		 2315 for (i = 0; i < chs->num; i++)
2316 mlx5e_activate_channel(chs->c[i]);
2317}
2318
2319static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2320{
2321 int err = 0;
2322 int i;
2323
1e7477ae
EBE		 2324 for (i = 0; i < chs->num; i++)
2325 err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq,
2326 err ? 0 : 20000);
acc6c595 2327
1e7477ae 2328 return err ? -ETIMEDOUT : 0;
acc6c595
SM		 2329}
2330
2331static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
2332{
2333 int i;
2334
2335 for (i = 0; i < chs->num; i++)
2336 mlx5e_deactivate_channel(chs->c[i]);
2337}
2338
55c2503d 2339void mlx5e_close_channels(struct mlx5e_channels *chs)
acc6c595
SM		 2340{
2341 int i;
c3b7c5c9 2342
ff9c852f
SM		 2343 for (i = 0; i < chs->num; i++)
2344 mlx5e_close_channel(chs->c[i]);
f62b8bb8 2345
ff9c852f
SM		 2346 kfree(chs->c);
2347 chs->num = 0;
f62b8bb8
AV		 2348}
2349
a5f97fee
SM		 2350static int
2351mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, struct mlx5e_rqt *rqt)
f62b8bb8
AV		 2352{
2353 struct mlx5_core_dev *mdev = priv->mdev;
f62b8bb8
AV		 2354 void *rqtc;
2355 int inlen;
2356 int err;
1da36696 2357 u32 *in;
a5f97fee 2358 int i;
f62b8bb8 2359
f62b8bb8 2360 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1b9a07ee 2361 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 2362 if (!in)
2363 return -ENOMEM;
2364
2365 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2366
2367 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2368 MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2369
a5f97fee
SM		 2370 for (i = 0; i < sz; i++)
2371 MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2be6967c 2372
398f3351
HHZ		 2373 err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
2374 if (!err)
2375 rqt->enabled = true;
f62b8bb8
AV		 2376
2377 kvfree(in);
1da36696
TT		 2378 return err;
2379}
2380
cb67b832 2381void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
1da36696 2382{
398f3351
HHZ		 2383 rqt->enabled = false;
2384 mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
1da36696
TT		 2385}
2386
8f493ffd 2387int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv)
6bfd390b
HHZ		 2388{
2389 struct mlx5e_rqt *rqt = &priv->indir_rqt;
8f493ffd 2390 int err;
6bfd390b 2391
8f493ffd
SM		 2392 err = mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, rqt);
2393 if (err)
2394 mlx5_core_warn(priv->mdev, "create indirect rqts failed, %d\n", err);
2395 return err;
6bfd390b
HHZ		 2396}
2397
cb67b832 2398int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
1da36696 2399{
398f3351 2400 struct mlx5e_rqt *rqt;
1da36696
TT		 2401 int err;
2402 int ix;
2403
6bfd390b 2404 for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
398f3351 2405 rqt = &priv->direct_tir[ix].rqt;
a5f97fee 2406 err = mlx5e_create_rqt(priv, 1 /*size */, rqt);
1da36696
TT		 2407 if (err)
2408 goto err_destroy_rqts;
2409 }
2410
2411 return 0;
2412
2413err_destroy_rqts:
8f493ffd 2414 mlx5_core_warn(priv->mdev, "create direct rqts failed, %d\n", err);
1da36696 2415 for (ix--; ix >= 0; ix--)
398f3351 2416 mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt);
1da36696 2417
f62b8bb8
AV		 2418 return err;
2419}
2420
8f493ffd
SM		 2421void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv)
2422{
2423 int i;
2424
2425 for (i = 0; i < priv->profile->max_nch(priv->mdev); i++)
2426 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
2427}
2428
a5f97fee
SM		 2429static int mlx5e_rx_hash_fn(int hfunc)
2430{
2431 return (hfunc == ETH_RSS_HASH_TOP) ?
2432 MLX5_RX_HASH_FN_TOEPLITZ :
2433 MLX5_RX_HASH_FN_INVERTED_XOR8;
2434}
2435
3f6d08d1 2436int mlx5e_bits_invert(unsigned long a, int size)
a5f97fee
SM		 2437{
2438 int inv = 0;
2439 int i;
2440
2441 for (i = 0; i < size; i++)
2442 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
2443
2444 return inv;
2445}
2446
2447static void mlx5e_fill_rqt_rqns(struct mlx5e_priv *priv, int sz,
2448 struct mlx5e_redirect_rqt_param rrp, void *rqtc)
2449{
2450 int i;
2451
2452 for (i = 0; i < sz; i++) {
2453 u32 rqn;
2454
2455 if (rrp.is_rss) {
2456 int ix = i;
2457
2458 if (rrp.rss.hfunc == ETH_RSS_HASH_XOR)
2459 ix = mlx5e_bits_invert(i, ilog2(sz));
2460
6a9764ef 2461 ix = priv->channels.params.indirection_rqt[ix];
a5f97fee
SM		 2462 rqn = rrp.rss.channels->c[ix]->rq.rqn;
2463 } else {
2464 rqn = rrp.rqn;
2465 }
2466 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
2467 }
2468}
2469
2470int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
2471 struct mlx5e_redirect_rqt_param rrp)
5c50368f
AS		 2472{
2473 struct mlx5_core_dev *mdev = priv->mdev;
5c50368f
AS		 2474 void *rqtc;
2475 int inlen;
1da36696 2476 u32 *in;
5c50368f
AS		 2477 int err;
2478
5c50368f 2479 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
1b9a07ee 2480 in = kvzalloc(inlen, GFP_KERNEL);
5c50368f
AS		 2481 if (!in)
2482 return -ENOMEM;
2483
2484 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2485
2486 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
5c50368f 2487 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
a5f97fee 2488 mlx5e_fill_rqt_rqns(priv, sz, rrp, rqtc);
1da36696 2489 err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
5c50368f
AS		 2490
2491 kvfree(in);
5c50368f
AS		 2492 return err;
2493}
2494
a5f97fee
SM		 2495static u32 mlx5e_get_direct_rqn(struct mlx5e_priv *priv, int ix,
2496 struct mlx5e_redirect_rqt_param rrp)
2497{
2498 if (!rrp.is_rss)
2499 return rrp.rqn;
2500
2501 if (ix >= rrp.rss.channels->num)
2502 return priv->drop_rq.rqn;
2503
2504 return rrp.rss.channels->c[ix]->rq.rqn;
2505}
2506
2507static void mlx5e_redirect_rqts(struct mlx5e_priv *priv,
2508 struct mlx5e_redirect_rqt_param rrp)
40ab6a6e 2509{
1da36696
TT		 2510 u32 rqtn;
2511 int ix;
2512
398f3351 2513 if (priv->indir_rqt.enabled) {
a5f97fee 2514 /* RSS RQ table */
398f3351 2515 rqtn = priv->indir_rqt.rqtn;
a5f97fee 2516 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp);
398f3351
HHZ		 2517 }
2518
a5f97fee
SM		 2519 for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
2520 struct mlx5e_redirect_rqt_param direct_rrp = {
2521 .is_rss = false,
95632791
AM		 2522 {
2523 .rqn = mlx5e_get_direct_rqn(priv, ix, rrp)
2524 },
a5f97fee
SM		 2525 };
2526
2527 /* Direct RQ Tables */
398f3351
HHZ		 2528 if (!priv->direct_tir[ix].rqt.enabled)
2529 continue;
a5f97fee 2530
398f3351 2531 rqtn = priv->direct_tir[ix].rqt.rqtn;
a5f97fee 2532 mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp);
1da36696 2533 }
40ab6a6e
AS		 2534}
2535
a5f97fee
SM		 2536static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv *priv,
2537 struct mlx5e_channels *chs)
2538{
2539 struct mlx5e_redirect_rqt_param rrp = {
2540 .is_rss = true,
95632791
AM		 2541 {
2542 .rss = {
2543 .channels = chs,
2544 .hfunc = chs->params.rss_hfunc,
2545 }
2546 },
a5f97fee
SM		 2547 };
2548
2549 mlx5e_redirect_rqts(priv, rrp);
2550}
2551
2552static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv *priv)
2553{
2554 struct mlx5e_redirect_rqt_param drop_rrp = {
2555 .is_rss = false,
95632791
AM		 2556 {
2557 .rqn = priv->drop_rq.rqn,
2558 },
a5f97fee
SM		 2559 };
2560
2561 mlx5e_redirect_rqts(priv, drop_rrp);
2562}
2563
6a9764ef 2564static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc)
5c50368f 2565{
6a9764ef 2566 if (!params->lro_en)
5c50368f
AS		 2567 return;
2568
2569#define ROUGH_MAX_L2_L3_HDR_SZ 256
2570
2571 MLX5_SET(tirc, tirc, lro_enable_mask,
2572 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2573 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2574 MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
6a9764ef
SM		 2575 (params->lro_wqe_sz - ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2576 MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout);
5c50368f
AS		 2577}
2578
6a9764ef
SM		 2579void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_params *params,
2580 enum mlx5e_traffic_types tt,
7b3722fa 2581 void *tirc, bool inner)
bdfc028d 2582{
7b3722fa
GP		 2583 void *hfso = inner ? MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner) :
2584 MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
a100ff3e
GP		 2585
2586#define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\
2587 MLX5_HASH_FIELD_SEL_DST_IP)
2588
2589#define MLX5_HASH_IP_L4PORTS (MLX5_HASH_FIELD_SEL_SRC_IP |\
2590 MLX5_HASH_FIELD_SEL_DST_IP |\
2591 MLX5_HASH_FIELD_SEL_L4_SPORT |\
2592 MLX5_HASH_FIELD_SEL_L4_DPORT)
2593
2594#define MLX5_HASH_IP_IPSEC_SPI (MLX5_HASH_FIELD_SEL_SRC_IP |\
2595 MLX5_HASH_FIELD_SEL_DST_IP |\
2596 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2597
6a9764ef
SM		 2598 MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_rx_hash_fn(params->rss_hfunc));
2599 if (params->rss_hfunc == ETH_RSS_HASH_TOP) {
bdfc028d
TT		 2600 void *rss_key = MLX5_ADDR_OF(tirc, tirc,
2601 rx_hash_toeplitz_key);
2602 size_t len = MLX5_FLD_SZ_BYTES(tirc,
2603 rx_hash_toeplitz_key);
2604
2605 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
6a9764ef 2606 memcpy(rss_key, params->toeplitz_hash_key, len);
bdfc028d 2607 }
a100ff3e
GP		 2608
2609 switch (tt) {
2610 case MLX5E_TT_IPV4_TCP:
2611 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2612 MLX5_L3_PROT_TYPE_IPV4);
2613 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2614 MLX5_L4_PROT_TYPE_TCP);
2615 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2616 MLX5_HASH_IP_L4PORTS);
2617 break;
2618
2619 case MLX5E_TT_IPV6_TCP:
2620 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2621 MLX5_L3_PROT_TYPE_IPV6);
2622 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2623 MLX5_L4_PROT_TYPE_TCP);
2624 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2625 MLX5_HASH_IP_L4PORTS);
2626 break;
2627
2628 case MLX5E_TT_IPV4_UDP:
2629 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2630 MLX5_L3_PROT_TYPE_IPV4);
2631 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2632 MLX5_L4_PROT_TYPE_UDP);
2633 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2634 MLX5_HASH_IP_L4PORTS);
2635 break;
2636
2637 case MLX5E_TT_IPV6_UDP:
2638 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2639 MLX5_L3_PROT_TYPE_IPV6);
2640 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2641 MLX5_L4_PROT_TYPE_UDP);
2642 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2643 MLX5_HASH_IP_L4PORTS);
2644 break;
2645
2646 case MLX5E_TT_IPV4_IPSEC_AH:
2647 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2648 MLX5_L3_PROT_TYPE_IPV4);
2649 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2650 MLX5_HASH_IP_IPSEC_SPI);
2651 break;
2652
2653 case MLX5E_TT_IPV6_IPSEC_AH:
2654 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2655 MLX5_L3_PROT_TYPE_IPV6);
2656 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2657 MLX5_HASH_IP_IPSEC_SPI);
2658 break;
2659
2660 case MLX5E_TT_IPV4_IPSEC_ESP:
2661 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2662 MLX5_L3_PROT_TYPE_IPV4);
2663 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2664 MLX5_HASH_IP_IPSEC_SPI);
2665 break;
2666
2667 case MLX5E_TT_IPV6_IPSEC_ESP:
2668 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2669 MLX5_L3_PROT_TYPE_IPV6);
2670 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2671 MLX5_HASH_IP_IPSEC_SPI);
2672 break;
2673
2674 case MLX5E_TT_IPV4:
2675 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2676 MLX5_L3_PROT_TYPE_IPV4);
2677 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2678 MLX5_HASH_IP);
2679 break;
2680
2681 case MLX5E_TT_IPV6:
2682 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2683 MLX5_L3_PROT_TYPE_IPV6);
2684 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2685 MLX5_HASH_IP);
2686 break;
2687 default:
2688 WARN_ONCE(true, "%s: bad traffic type!\n", __func__);
2689 }
bdfc028d
TT		 2690}
2691
ab0394fe 2692static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
5c50368f
AS		 2693{
2694 struct mlx5_core_dev *mdev = priv->mdev;
2695
2696 void *in;
2697 void *tirc;
2698 int inlen;
2699 int err;
ab0394fe 2700 int tt;
1da36696 2701 int ix;
5c50368f
AS		 2702
2703 inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
1b9a07ee 2704 in = kvzalloc(inlen, GFP_KERNEL);
5c50368f
AS		 2705 if (!in)
2706 return -ENOMEM;
2707
2708 MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
2709 tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
2710
6a9764ef 2711 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
5c50368f 2712
1da36696 2713 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
724b2aa1 2714 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in,
1da36696 2715 inlen);
ab0394fe 2716 if (err)
1da36696 2717 goto free_in;
ab0394fe 2718 }
5c50368f 2719
6bfd390b 2720 for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
1da36696
TT		 2721 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn,
2722 in, inlen);
2723 if (err)
2724 goto free_in;
2725 }
2726
2727free_in:
5c50368f
AS		 2728 kvfree(in);
2729
2730 return err;
2731}
2732
7b3722fa
GP		 2733static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv,
2734 enum mlx5e_traffic_types tt,
2735 u32 *tirc)
2736{
2737 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2738
2739 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
2740
2741 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2742 MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
2743 MLX5_SET(tirc, tirc, tunneled_offload_en, 0x1);
2744
2745 mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, true);
2746}
2747
472a1e44
TT		 2748static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
2749 struct mlx5e_params *params, u16 mtu)
40ab6a6e 2750{
472a1e44 2751 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu);
40ab6a6e
AS		 2752 int err;
2753
cd255eff 2754 err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
40ab6a6e
AS		 2755 if (err)
2756 return err;
2757
cd255eff
SM		 2758 /* Update vport context MTU */
2759 mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2760 return 0;
2761}
40ab6a6e 2762
472a1e44
TT		 2763static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
2764 struct mlx5e_params *params, u16 *mtu)
cd255eff 2765{
cd255eff
SM		 2766 u16 hw_mtu = 0;
2767 int err;
40ab6a6e 2768
cd255eff
SM		 2769 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2770 if (err || !hw_mtu) /* fallback to port oper mtu */
2771 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2772
472a1e44 2773 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu);
cd255eff
SM		 2774}
2775
2e20a151 2776static int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
cd255eff 2777{
472a1e44 2778 struct mlx5e_params *params = &priv->channels.params;
2e20a151 2779 struct net_device *netdev = priv->netdev;
472a1e44 2780 struct mlx5_core_dev *mdev = priv->mdev;
cd255eff
SM		 2781 u16 mtu;
2782 int err;
2783
472a1e44 2784 err = mlx5e_set_mtu(mdev, params, params->sw_mtu);
cd255eff
SM		 2785 if (err)
2786 return err;
40ab6a6e 2787
472a1e44
TT		 2788 mlx5e_query_mtu(mdev, params, &mtu);
2789 if (mtu != params->sw_mtu)
cd255eff 2790 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
472a1e44 2791 __func__, mtu, params->sw_mtu);
40ab6a6e 2792
472a1e44 2793 params->sw_mtu = mtu;
40ab6a6e
AS		 2794 return 0;
2795}
2796
08fb1dac
SM		 2797static void mlx5e_netdev_set_tcs(struct net_device *netdev)
2798{
2799 struct mlx5e_priv *priv = netdev_priv(netdev);
6a9764ef
SM		 2800 int nch = priv->channels.params.num_channels;
2801 int ntc = priv->channels.params.num_tc;
08fb1dac
SM		 2802 int tc;
2803
2804 netdev_reset_tc(netdev);
2805
2806 if (ntc == 1)
2807 return;
2808
2809 netdev_set_num_tc(netdev, ntc);
2810
7ccdd084
RS		 2811 /* Map netdev TCs to offset 0
2812 * We have our own UP to TXQ mapping for QoS
2813 */
08fb1dac 2814 for (tc = 0; tc < ntc; tc++)
7ccdd084 2815 netdev_set_tc_queue(netdev, tc, nch, 0);
08fb1dac
SM		 2816}
2817
8bfaf07f 2818static void mlx5e_build_tc2txq_maps(struct mlx5e_priv *priv)
acc6c595 2819{
8bfaf07f 2820 int max_nch = priv->profile->max_nch(priv->mdev);
acc6c595
SM		 2821 int i, tc;
2822
8bfaf07f 2823 for (i = 0; i < max_nch; i++)
acc6c595 2824 for (tc = 0; tc < priv->profile->max_tc; tc++)
8bfaf07f
EBE		 2825 priv->channel_tc2txq[i][tc] = i + tc * max_nch;
2826}
2827
2828static void mlx5e_build_tx2sq_maps(struct mlx5e_priv *priv)
2829{
2830 struct mlx5e_channel *c;
2831 struct mlx5e_txqsq *sq;
2832 int i, tc;
acc6c595
SM		 2833
2834 for (i = 0; i < priv->channels.num; i++) {
2835 c = priv->channels.c[i];
2836 for (tc = 0; tc < c->num_tc; tc++) {
2837 sq = &c->sq[tc];
2838 priv->txq2sq[sq->txq_ix] = sq;
2839 }
2840 }
2841}
2842
603f4a45 2843void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
acc6c595 2844{
9008ae07
SM		 2845 int num_txqs = priv->channels.num * priv->channels.params.num_tc;
2846 struct net_device *netdev = priv->netdev;
2847
2848 mlx5e_netdev_set_tcs(netdev);
053ee0a7
TR		 2849 netif_set_real_num_tx_queues(netdev, num_txqs);
2850 netif_set_real_num_rx_queues(netdev, priv->channels.num);
9008ae07 2851
8bfaf07f 2852 mlx5e_build_tx2sq_maps(priv);
acc6c595
SM		 2853 mlx5e_activate_channels(&priv->channels);
2854 netif_tx_start_all_queues(priv->netdev);
9008ae07 2855
733d3e54 2856 if (MLX5_ESWITCH_MANAGER(priv->mdev))
9008ae07
SM		 2857 mlx5e_add_sqs_fwd_rules(priv);
2858
acc6c595 2859 mlx5e_wait_channels_min_rx_wqes(&priv->channels);
9008ae07 2860 mlx5e_redirect_rqts_to_channels(priv, &priv->channels);
acc6c595
SM		 2861}
2862
603f4a45 2863void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
acc6c595 2864{
9008ae07
SM		 2865 mlx5e_redirect_rqts_to_drop(priv);
2866
733d3e54 2867 if (MLX5_ESWITCH_MANAGER(priv->mdev))
9008ae07
SM		 2868 mlx5e_remove_sqs_fwd_rules(priv);
2869
acc6c595
SM		 2870 /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
2871 * polling for inactive tx queues.
2872 */
2873 netif_tx_stop_all_queues(priv->netdev);
2874 netif_tx_disable(priv->netdev);
2875 mlx5e_deactivate_channels(&priv->channels);
2876}
2877
55c2503d 2878void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
2e20a151
SM		 2879 struct mlx5e_channels *new_chs,
2880 mlx5e_fp_hw_modify hw_modify)
55c2503d
SM		 2881{
2882 struct net_device *netdev = priv->netdev;
2883 int new_num_txqs;
7ca42c80 2884 int carrier_ok;
55c2503d
SM		 2885 new_num_txqs = new_chs->num * new_chs->params.num_tc;
2886
7ca42c80 2887 carrier_ok = netif_carrier_ok(netdev);
55c2503d
SM		 2888 netif_carrier_off(netdev);
2889
2890 if (new_num_txqs < netdev->real_num_tx_queues)
2891 netif_set_real_num_tx_queues(netdev, new_num_txqs);
2892
2893 mlx5e_deactivate_priv_channels(priv);
2894 mlx5e_close_channels(&priv->channels);
2895
2896 priv->channels = *new_chs;
2897
2e20a151
SM		 2898 /* New channels are ready to roll, modify HW settings if needed */
2899 if (hw_modify)
2900 hw_modify(priv);
2901
55c2503d
SM		 2902 mlx5e_refresh_tirs(priv, false);
2903 mlx5e_activate_priv_channels(priv);
2904
7ca42c80
ES		 2905 /* return carrier back if needed */
2906 if (carrier_ok)
2907 netif_carrier_on(netdev);
55c2503d
SM		 2908}
2909
237f258c 2910void mlx5e_timestamp_init(struct mlx5e_priv *priv)
7c39afb3
FD		 2911{
2912 priv->tstamp.tx_type = HWTSTAMP_TX_OFF;
2913 priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE;
2914}
2915
40ab6a6e
AS		 2916int mlx5e_open_locked(struct net_device *netdev)
2917{
2918 struct mlx5e_priv *priv = netdev_priv(netdev);
40ab6a6e
AS		 2919 int err;
2920
2921 set_bit(MLX5E_STATE_OPENED, &priv->state);
2922
ff9c852f 2923 err = mlx5e_open_channels(priv, &priv->channels);
acc6c595 2924 if (err)
343b29f3 2925 goto err_clear_state_opened_flag;
40ab6a6e 2926
b676f653 2927 mlx5e_refresh_tirs(priv, false);
acc6c595 2928 mlx5e_activate_priv_channels(priv);
7ca42c80
ES		 2929 if (priv->profile->update_carrier)
2930 priv->profile->update_carrier(priv);
be4891af 2931
cb67b832
HHZ		 2932 if (priv->profile->update_stats)
2933 queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
40ab6a6e 2934
9b37b07f 2935 return 0;
343b29f3
AS		 2936
2937err_clear_state_opened_flag:
2938 clear_bit(MLX5E_STATE_OPENED, &priv->state);
2939 return err;
40ab6a6e
AS		 2940}
2941
cb67b832 2942int mlx5e_open(struct net_device *netdev)
40ab6a6e
AS		 2943{
2944 struct mlx5e_priv *priv = netdev_priv(netdev);
2945 int err;
2946
2947 mutex_lock(&priv->state_lock);
2948 err = mlx5e_open_locked(netdev);
63bfd399
EBE		 2949 if (!err)
2950 mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_UP);
40ab6a6e
AS		 2951 mutex_unlock(&priv->state_lock);
2952
a117f73d
SK		 2953 if (mlx5e_vxlan_allowed(priv->mdev))
2954 udp_tunnel_get_rx_info(netdev);
2955
40ab6a6e
AS		 2956 return err;
2957}
2958
2959int mlx5e_close_locked(struct net_device *netdev)
2960{
2961 struct mlx5e_priv *priv = netdev_priv(netdev);
2962
a1985740
AS		 2963 /* May already be CLOSED in case a previous configuration operation
2964 * (e.g RX/TX queue size change) that involves close&open failed.
2965 */
2966 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2967 return 0;
2968
40ab6a6e
AS		 2969 clear_bit(MLX5E_STATE_OPENED, &priv->state);
2970
40ab6a6e 2971 netif_carrier_off(priv->netdev);
acc6c595
SM		 2972 mlx5e_deactivate_priv_channels(priv);
2973 mlx5e_close_channels(&priv->channels);
40ab6a6e
AS		 2974
2975 return 0;
2976}
2977
cb67b832 2978int mlx5e_close(struct net_device *netdev)
40ab6a6e
AS		 2979{
2980 struct mlx5e_priv *priv = netdev_priv(netdev);
2981 int err;
2982
26e59d80
MHY		 2983 if (!netif_device_present(netdev))
2984 return -ENODEV;
2985
40ab6a6e 2986 mutex_lock(&priv->state_lock);
63bfd399 2987 mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_DOWN);
40ab6a6e
AS		 2988 err = mlx5e_close_locked(netdev);
2989 mutex_unlock(&priv->state_lock);
2990
2991 return err;
2992}
2993
a43b25da 2994static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
3b77235b
SM		 2995 struct mlx5e_rq *rq,
2996 struct mlx5e_rq_param *param)
40ab6a6e 2997{
40ab6a6e
AS		 2998 void *rqc = param->rqc;
2999 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
3000 int err;
3001
3002 param->wq.db_numa_node = param->wq.buf_numa_node;
3003
99cbfa93
TT		 3004 err = mlx5_wq_cyc_create(mdev, &param->wq, rqc_wq, &rq->wqe.wq,
3005 &rq->wq_ctrl);
40ab6a6e
AS		 3006 if (err)
3007 return err;
3008
0ddf5432
JDB		 3009 /* Mark as unused given "Drop-RQ" packets never reach XDP */
3010 xdp_rxq_info_unused(&rq->xdp_rxq);
3011
a43b25da 3012 rq->mdev = mdev;
40ab6a6e
AS		 3013
3014 return 0;
3015}
3016
a43b25da 3017static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev,
3b77235b
SM		 3018 struct mlx5e_cq *cq,
3019 struct mlx5e_cq_param *param)
40ab6a6e 3020{
2f0db879
GP		 3021 param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev);
3022 param->wq.db_numa_node = dev_to_node(&mdev->pdev->dev);
3023
95b6c6a5 3024 return mlx5e_alloc_cq_common(mdev, param, cq);
40ab6a6e
AS		 3025}
3026
7cbaf9a3 3027static int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
a43b25da 3028 struct mlx5e_rq *drop_rq)
40ab6a6e 3029{
7cbaf9a3 3030 struct mlx5_core_dev *mdev = priv->mdev;
a43b25da
SM		 3031 struct mlx5e_cq_param cq_param = {};
3032 struct mlx5e_rq_param rq_param = {};
3033 struct mlx5e_cq *cq = &drop_rq->cq;
40ab6a6e
AS		 3034 int err;
3035
7cbaf9a3 3036 mlx5e_build_drop_rq_param(priv, &rq_param);
40ab6a6e 3037
a43b25da 3038 err = mlx5e_alloc_drop_cq(mdev, cq, &cq_param);
40ab6a6e
AS		 3039 if (err)
3040 return err;
3041
3b77235b 3042 err = mlx5e_create_cq(cq, &cq_param);
40ab6a6e 3043 if (err)
3b77235b 3044 goto err_free_cq;
40ab6a6e 3045
a43b25da 3046 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
40ab6a6e 3047 if (err)
3b77235b 3048 goto err_destroy_cq;
40ab6a6e 3049
a43b25da 3050 err = mlx5e_create_rq(drop_rq, &rq_param);
40ab6a6e 3051 if (err)
3b77235b 3052 goto err_free_rq;
40ab6a6e 3053
7cbaf9a3
MS		 3054 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3055 if (err)
3056 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err);
3057
40ab6a6e
AS		 3058 return 0;
3059
3b77235b 3060err_free_rq:
a43b25da 3061 mlx5e_free_rq(drop_rq);
40ab6a6e
AS		 3062
3063err_destroy_cq:
a43b25da 3064 mlx5e_destroy_cq(cq);
40ab6a6e 3065
3b77235b 3066err_free_cq:
a43b25da 3067 mlx5e_free_cq(cq);
3b77235b 3068
40ab6a6e
AS		 3069 return err;
3070}
3071
a43b25da 3072static void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
40ab6a6e 3073{
a43b25da
SM		 3074 mlx5e_destroy_rq(drop_rq);
3075 mlx5e_free_rq(drop_rq);
3076 mlx5e_destroy_cq(&drop_rq->cq);
3077 mlx5e_free_cq(&drop_rq->cq);
40ab6a6e
AS		 3078}
3079
5426a0b2
SM		 3080int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc,
3081 u32 underlay_qpn, u32 *tisn)
40ab6a6e 3082{
c4f287c4 3083 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
40ab6a6e
AS		 3084 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3085
08fb1dac 3086 MLX5_SET(tisc, tisc, prio, tc << 1);
5426a0b2 3087 MLX5_SET(tisc, tisc, underlay_qpn, underlay_qpn);
b50d292b 3088 MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
db60b802
AH		 3089
3090 if (mlx5_lag_is_lacp_owner(mdev))
3091 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
3092
5426a0b2 3093 return mlx5_core_create_tis(mdev, in, sizeof(in), tisn);
40ab6a6e
AS		 3094}
3095
5426a0b2 3096void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn)
40ab6a6e 3097{
5426a0b2 3098 mlx5_core_destroy_tis(mdev, tisn);
40ab6a6e
AS		 3099}
3100
cb67b832 3101int mlx5e_create_tises(struct mlx5e_priv *priv)
40ab6a6e
AS		 3102{
3103 int err;
3104 int tc;
3105
6bfd390b 3106 for (tc = 0; tc < priv->profile->max_tc; tc++) {
5426a0b2 3107 err = mlx5e_create_tis(priv->mdev, tc, 0, &priv->tisn[tc]);
40ab6a6e
AS		 3108 if (err)
3109 goto err_close_tises;
3110 }
3111
3112 return 0;
3113
3114err_close_tises:
3115 for (tc--; tc >= 0; tc--)
5426a0b2 3116 mlx5e_destroy_tis(priv->mdev, priv->tisn[tc]);
40ab6a6e
AS		 3117
3118 return err;
3119}
3120
cb67b832 3121void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
40ab6a6e
AS		 3122{
3123 int tc;
3124
6bfd390b 3125 for (tc = 0; tc < priv->profile->max_tc; tc++)
5426a0b2 3126 mlx5e_destroy_tis(priv->mdev, priv->tisn[tc]);
40ab6a6e
AS		 3127}
3128
6a9764ef
SM		 3129static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv,
3130 enum mlx5e_traffic_types tt,
3131 u32 *tirc)
f62b8bb8 3132{
b50d292b 3133 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
3191e05f 3134
6a9764ef 3135 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
f62b8bb8 3136
4cbeaff5 3137 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
398f3351 3138 MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
7b3722fa 3139 mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, false);
f62b8bb8
AV		 3140}
3141
6a9764ef 3142static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc)
f62b8bb8 3143{
b50d292b 3144 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
1da36696 3145
6a9764ef 3146 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
1da36696
TT		 3147
3148 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
3149 MLX5_SET(tirc, tirc, indirect_table, rqtn);
3150 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
3151}
3152
8f493ffd 3153int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
1da36696 3154{
724b2aa1 3155 struct mlx5e_tir *tir;
f62b8bb8
AV		 3156 void *tirc;
3157 int inlen;
7b3722fa 3158 int i = 0;
f62b8bb8 3159 int err;
1da36696 3160 u32 *in;
1da36696 3161 int tt;
f62b8bb8
AV		 3162
3163 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 3164 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 3165 if (!in)
3166 return -ENOMEM;
3167
1da36696
TT		 3168 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
3169 memset(in, 0, inlen);
724b2aa1 3170 tir = &priv->indir_tir[tt];
1da36696 3171 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
6a9764ef 3172 mlx5e_build_indir_tir_ctx(priv, tt, tirc);
724b2aa1 3173 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
7b3722fa
GP		 3174 if (err) {
3175 mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err);
3176 goto err_destroy_inner_tirs;
3177 }
f62b8bb8
AV		 3178 }
3179
7b3722fa
GP		 3180 if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
3181 goto out;
3182
3183 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) {
3184 memset(in, 0, inlen);
3185 tir = &priv->inner_indir_tir[i];
3186 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3187 mlx5e_build_inner_indir_tir_ctx(priv, i, tirc);
3188 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
3189 if (err) {
3190 mlx5_core_warn(priv->mdev, "create inner indirect tirs failed, %d\n", err);
3191 goto err_destroy_inner_tirs;
3192 }
3193 }
3194
3195out:
6bfd390b
HHZ		 3196 kvfree(in);
3197
3198 return 0;
3199
7b3722fa
GP		 3200err_destroy_inner_tirs:
3201 for (i--; i >= 0; i--)
3202 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
3203
6bfd390b
HHZ		 3204 for (tt--; tt >= 0; tt--)
3205 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
3206
3207 kvfree(in);
3208
3209 return err;
3210}
3211
cb67b832 3212int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)
6bfd390b
HHZ		 3213{
3214 int nch = priv->profile->max_nch(priv->mdev);
3215 struct mlx5e_tir *tir;
3216 void *tirc;
3217 int inlen;
3218 int err;
3219 u32 *in;
3220 int ix;
3221
3222 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 3223 in = kvzalloc(inlen, GFP_KERNEL);
6bfd390b
HHZ		 3224 if (!in)
3225 return -ENOMEM;
3226
1da36696
TT		 3227 for (ix = 0; ix < nch; ix++) {
3228 memset(in, 0, inlen);
724b2aa1 3229 tir = &priv->direct_tir[ix];
1da36696 3230 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
6a9764ef 3231 mlx5e_build_direct_tir_ctx(priv, priv->direct_tir[ix].rqt.rqtn, tirc);
724b2aa1 3232 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
1da36696
TT		 3233 if (err)
3234 goto err_destroy_ch_tirs;
3235 }
3236
3237 kvfree(in);
3238
f62b8bb8
AV		 3239 return 0;
3240
1da36696 3241err_destroy_ch_tirs:
8f493ffd 3242 mlx5_core_warn(priv->mdev, "create direct tirs failed, %d\n", err);
1da36696 3243 for (ix--; ix >= 0; ix--)
724b2aa1 3244 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]);
1da36696 3245
1da36696 3246 kvfree(in);
f62b8bb8
AV		 3247
3248 return err;
3249}
3250
8f493ffd 3251void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
f62b8bb8
AV		 3252{
3253 int i;
3254
1da36696 3255 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
724b2aa1 3256 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
7b3722fa
GP		 3257
3258 if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
3259 return;
3260
3261 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
3262 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
f62b8bb8
AV		 3263}
3264
cb67b832 3265void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
6bfd390b
HHZ		 3266{
3267 int nch = priv->profile->max_nch(priv->mdev);
3268 int i;
3269
3270 for (i = 0; i < nch; i++)
3271 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);
3272}
3273
102722fc
GE		 3274static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable)
3275{
3276 int err = 0;
3277 int i;
3278
3279 for (i = 0; i < chs->num; i++) {
3280 err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable);
3281 if (err)
3282 return err;
3283 }
3284
3285 return 0;
3286}
3287
f6d96a20 3288static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
36350114
GP		 3289{
3290 int err = 0;
3291 int i;
3292
ff9c852f
SM		 3293 for (i = 0; i < chs->num; i++) {
3294 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
36350114
GP		 3295 if (err)
3296 return err;
3297 }
3298
3299 return 0;
3300}
3301
0cf0f6d3
JP		 3302static int mlx5e_setup_tc_mqprio(struct net_device *netdev,
3303 struct tc_mqprio_qopt *mqprio)
08fb1dac
SM		 3304{
3305 struct mlx5e_priv *priv = netdev_priv(netdev);
6f9485af 3306 struct mlx5e_channels new_channels = {};
0cf0f6d3 3307 u8 tc = mqprio->num_tc;
08fb1dac
SM		 3308 int err = 0;
3309
0cf0f6d3
JP		 3310 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
3311
08fb1dac
SM		 3312 if (tc && tc != MLX5E_MAX_NUM_TC)
3313 return -EINVAL;
3314
3315 mutex_lock(&priv->state_lock);
3316
6f9485af
SM		 3317 new_channels.params = priv->channels.params;
3318 new_channels.params.num_tc = tc ? tc : 1;
08fb1dac 3319
20b6a1c7 3320 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
6f9485af
SM		 3321 priv->channels.params = new_channels.params;
3322 goto out;
3323 }
08fb1dac 3324
6f9485af
SM		 3325 err = mlx5e_open_channels(priv, &new_channels);
3326 if (err)
3327 goto out;
08fb1dac 3328
05909bab
EBE		 3329 priv->max_opened_tc = max_t(u8, priv->max_opened_tc,
3330 new_channels.params.num_tc);
2e20a151 3331 mlx5e_switch_priv_channels(priv, &new_channels, NULL);
6f9485af 3332out:
08fb1dac 3333 mutex_unlock(&priv->state_lock);
08fb1dac
SM		 3334 return err;
3335}
3336
e80541ec 3337#ifdef CONFIG_MLX5_ESWITCH
d6c862ba 3338static int mlx5e_setup_tc_cls_flower(struct mlx5e_priv *priv,
60bd4af8
OG		 3339 struct tc_cls_flower_offload *cls_flower,
3340 int flags)
08fb1dac 3341{
0cf0f6d3
JP		 3342 switch (cls_flower->command) {
3343 case TC_CLSFLOWER_REPLACE:
60bd4af8 3344 return mlx5e_configure_flower(priv, cls_flower, flags);
0cf0f6d3 3345 case TC_CLSFLOWER_DESTROY:
60bd4af8 3346 return mlx5e_delete_flower(priv, cls_flower, flags);
0cf0f6d3 3347 case TC_CLSFLOWER_STATS:
60bd4af8 3348 return mlx5e_stats_flower(priv, cls_flower, flags);
0cf0f6d3 3349 default:
a5fcf8a6 3350 return -EOPNOTSUPP;
0cf0f6d3
JP		 3351 }
3352}
d6c862ba 3353
60bd4af8
OG		 3354static int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
3355 void *cb_priv)
d6c862ba
JP		 3356{
3357 struct mlx5e_priv *priv = cb_priv;
3358
9ab88e83 3359 if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))
44ae12a7
JP		 3360 return -EOPNOTSUPP;
3361
d6c862ba
JP		 3362 switch (type) {
3363 case TC_SETUP_CLSFLOWER:
60bd4af8 3364 return mlx5e_setup_tc_cls_flower(priv, type_data, MLX5E_TC_INGRESS);
d6c862ba
JP		 3365 default:
3366 return -EOPNOTSUPP;
3367 }
3368}
3369
3370static int mlx5e_setup_tc_block(struct net_device *dev,
3371 struct tc_block_offload *f)
3372{
3373 struct mlx5e_priv *priv = netdev_priv(dev);
3374
3375 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
3376 return -EOPNOTSUPP;
3377
3378 switch (f->command) {
3379 case TC_BLOCK_BIND:
3380 return tcf_block_cb_register(f->block, mlx5e_setup_tc_block_cb,
60513bd8 3381 priv, priv, f->extack);
d6c862ba
JP		 3382 case TC_BLOCK_UNBIND:
3383 tcf_block_cb_unregister(f->block, mlx5e_setup_tc_block_cb,
3384 priv);
3385 return 0;
3386 default:
3387 return -EOPNOTSUPP;
3388 }
3389}
e80541ec 3390#endif
a5fcf8a6 3391
9afe9a53
OG		 3392static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
3393 void *type_data)
0cf0f6d3 3394{
2572ac53 3395 switch (type) {
fde6af47 3396#ifdef CONFIG_MLX5_ESWITCH
d6c862ba
JP		 3397 case TC_SETUP_BLOCK:
3398 return mlx5e_setup_tc_block(dev, type_data);
fde6af47 3399#endif
575ed7d3 3400 case TC_SETUP_QDISC_MQPRIO:
de4784ca 3401 return mlx5e_setup_tc_mqprio(dev, type_data);
e8f887ac
AV		 3402 default:
3403 return -EOPNOTSUPP;
3404 }
08fb1dac
SM		 3405}
3406
bc1f4470 3407static void
f62b8bb8
AV		 3408mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
3409{
3410 struct mlx5e_priv *priv = netdev_priv(dev);
9218b44d 3411 struct mlx5e_sw_stats *sstats = &priv->stats.sw;
f62b8bb8 3412 struct mlx5e_vport_stats *vstats = &priv->stats.vport;
269e6b3a 3413 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
f62b8bb8 3414
ed56c519
SM		 3415 /* update HW stats in background for next time */
3416 queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
3417
370bad0f
OG		 3418 if (mlx5e_is_uplink_rep(priv)) {
3419 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
3420 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok);
3421 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
3422 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
3423 } else {
868a01a2 3424 mlx5e_grp_sw_update_stats(priv);
370bad0f
OG		 3425 stats->rx_packets = sstats->rx_packets;
3426 stats->rx_bytes = sstats->rx_bytes;
3427 stats->tx_packets = sstats->tx_packets;
3428 stats->tx_bytes = sstats->tx_bytes;
3429 stats->tx_dropped = sstats->tx_queue_dropped;
3430 }
269e6b3a
GP		 3431
3432 stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
269e6b3a
GP		 3433
3434 stats->rx_length_errors =
9218b44d
GP		 3435 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
3436 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
3437 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
269e6b3a 3438 stats->rx_crc_errors =
9218b44d
GP		 3439 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
3440 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
3441 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
269e6b3a
GP		 3442 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
3443 stats->rx_frame_errors;
3444 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
3445
3446 /* vport multicast also counts packets that are dropped due to steering
3447 * or rx out of buffer
3448 */
9218b44d
GP		 3449 stats->multicast =
3450 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
f62b8bb8
AV		 3451}
3452
3453static void mlx5e_set_rx_mode(struct net_device *dev)
3454{
3455 struct mlx5e_priv *priv = netdev_priv(dev);
3456
7bb29755 3457 queue_work(priv->wq, &priv->set_rx_mode_work);
f62b8bb8
AV		 3458}
3459
3460static int mlx5e_set_mac(struct net_device *netdev, void *addr)
3461{
3462 struct mlx5e_priv *priv = netdev_priv(netdev);
3463 struct sockaddr *saddr = addr;
3464
3465 if (!is_valid_ether_addr(saddr->sa_data))
3466 return -EADDRNOTAVAIL;
3467
3468 netif_addr_lock_bh(netdev);
3469 ether_addr_copy(netdev->dev_addr, saddr->sa_data);
3470 netif_addr_unlock_bh(netdev);
3471
7bb29755 3472 queue_work(priv->wq, &priv->set_rx_mode_work);
f62b8bb8
AV		 3473
3474 return 0;
3475}
3476
75b81ce7 3477#define MLX5E_SET_FEATURE(features, feature, enable) \
0e405443
GP		 3478 do { \
3479 if (enable) \
75b81ce7 3480 *features |= feature; \
0e405443 3481 else \
75b81ce7 3482 *features &= ~feature; \
0e405443
GP		 3483 } while (0)
3484
3485typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3486
3487static int set_feature_lro(struct net_device *netdev, bool enable)
f62b8bb8
AV		 3488{
3489 struct mlx5e_priv *priv = netdev_priv(netdev);
619a8f2a 3490 struct mlx5_core_dev *mdev = priv->mdev;
2e20a151 3491 struct mlx5e_channels new_channels = {};
619a8f2a 3492 struct mlx5e_params *old_params;
2e20a151
SM		 3493 int err = 0;
3494 bool reset;
f62b8bb8
AV		 3495
3496 mutex_lock(&priv->state_lock);
f62b8bb8 3497
619a8f2a 3498 old_params = &priv->channels.params;
6c3a823e
TT		 3499 if (enable && !MLX5E_GET_PFLAG(old_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
3500 netdev_warn(netdev, "can't set LRO with legacy RQ\n");
3501 err = -EINVAL;
3502 goto out;
3503 }
3504
619a8f2a 3505 reset = test_bit(MLX5E_STATE_OPENED, &priv->state);
98e81b0a 3506
619a8f2a 3507 new_channels.params = *old_params;
2e20a151
SM		 3508 new_channels.params.lro_en = enable;
3509
99cbfa93 3510 if (old_params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) {
619a8f2a
TT		 3511 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, old_params) ==
3512 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_channels.params))
3513 reset = false;
3514 }
3515
2e20a151 3516 if (!reset) {
619a8f2a 3517 *old_params = new_channels.params;
2e20a151
SM		 3518 err = mlx5e_modify_tirs_lro(priv);
3519 goto out;
98e81b0a 3520 }
f62b8bb8 3521
2e20a151
SM		 3522 err = mlx5e_open_channels(priv, &new_channels);
3523 if (err)
3524 goto out;
0e405443 3525
2e20a151
SM		 3526 mlx5e_switch_priv_channels(priv, &new_channels, mlx5e_modify_tirs_lro);
3527out:
9b37b07f 3528 mutex_unlock(&priv->state_lock);
0e405443
GP		 3529 return err;
3530}
3531
2b52a283 3532static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
0e405443
GP		 3533{
3534 struct mlx5e_priv *priv = netdev_priv(netdev);
3535
3536 if (enable)
2b52a283 3537 mlx5e_enable_cvlan_filter(priv);
0e405443 3538 else
2b52a283 3539 mlx5e_disable_cvlan_filter(priv);
0e405443
GP		 3540
3541 return 0;
3542}
3543
3544static int set_feature_tc_num_filters(struct net_device *netdev, bool enable)
3545{
3546 struct mlx5e_priv *priv = netdev_priv(netdev);
f62b8bb8 3547
0e405443 3548 if (!enable && mlx5e_tc_num_filters(priv)) {
e8f887ac
AV		 3549 netdev_err(netdev,
3550 "Active offloaded tc filters, can't turn hw_tc_offload off\n");
3551 return -EINVAL;
3552 }
3553
0e405443
GP		 3554 return 0;
3555}
3556
94cb1ebb
EBE		 3557static int set_feature_rx_all(struct net_device *netdev, bool enable)
3558{
3559 struct mlx5e_priv *priv = netdev_priv(netdev);
3560 struct mlx5_core_dev *mdev = priv->mdev;
3561
3562 return mlx5_set_port_fcs(mdev, !enable);
3563}
3564
102722fc
GE		 3565static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
3566{
3567 struct mlx5e_priv *priv = netdev_priv(netdev);
3568 int err;
3569
3570 mutex_lock(&priv->state_lock);
3571
3572 priv->channels.params.scatter_fcs_en = enable;
3573 err = mlx5e_modify_channels_scatter_fcs(&priv->channels, enable);
3574 if (err)
3575 priv->channels.params.scatter_fcs_en = !enable;
3576
3577 mutex_unlock(&priv->state_lock);
3578
3579 return err;
3580}
3581
36350114
GP		 3582static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
3583{
3584 struct mlx5e_priv *priv = netdev_priv(netdev);
ff9c852f 3585 int err = 0;
36350114
GP		 3586
3587 mutex_lock(&priv->state_lock);
3588
6a9764ef 3589 priv->channels.params.vlan_strip_disable = !enable;
ff9c852f
SM		 3590 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3591 goto unlock;
3592
3593 err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
36350114 3594 if (err)
6a9764ef 3595 priv->channels.params.vlan_strip_disable = enable;
36350114 3596
ff9c852f 3597unlock:
36350114
GP		 3598 mutex_unlock(&priv->state_lock);
3599
3600 return err;
3601}
3602
45bf454a
MG		 3603#ifdef CONFIG_RFS_ACCEL
3604static int set_feature_arfs(struct net_device *netdev, bool enable)
3605{
3606 struct mlx5e_priv *priv = netdev_priv(netdev);
3607 int err;
3608
3609 if (enable)
3610 err = mlx5e_arfs_enable(priv);
3611 else
3612 err = mlx5e_arfs_disable(priv);
3613
3614 return err;
3615}
3616#endif
3617
0e405443 3618static int mlx5e_handle_feature(struct net_device *netdev,
75b81ce7 3619 netdev_features_t *features,
0e405443
GP		 3620 netdev_features_t wanted_features,
3621 netdev_features_t feature,
3622 mlx5e_feature_handler feature_handler)
3623{
3624 netdev_features_t changes = wanted_features ^ netdev->features;
3625 bool enable = !!(wanted_features & feature);
3626 int err;
3627
3628 if (!(changes & feature))
3629 return 0;
3630
3631 err = feature_handler(netdev, enable);
3632 if (err) {
b20eab15
GP		 3633 netdev_err(netdev, "%s feature %pNF failed, err %d\n",
3634 enable ? "Enable" : "Disable", &feature, err);
0e405443
GP		 3635 return err;
3636 }
3637
75b81ce7 3638 MLX5E_SET_FEATURE(features, feature, enable);
0e405443
GP		 3639 return 0;
3640}
3641
3642static int mlx5e_set_features(struct net_device *netdev,
3643 netdev_features_t features)
3644{
75b81ce7 3645 netdev_features_t oper_features = netdev->features;
be0f780b
GP		 3646 int err = 0;
3647
3648#define MLX5E_HANDLE_FEATURE(feature, handler) \
3649 mlx5e_handle_feature(netdev, &oper_features, features, feature, handler)
0e405443 3650
be0f780b
GP		 3651 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro);
3652 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER,
2b52a283 3653 set_feature_cvlan_filter);
be0f780b
GP		 3654 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_tc_num_filters);
3655 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all);
3656 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs);
3657 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan);
45bf454a 3658#ifdef CONFIG_RFS_ACCEL
be0f780b 3659 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs);
45bf454a 3660#endif
0e405443 3661
75b81ce7
GP		 3662 if (err) {
3663 netdev->features = oper_features;
3664 return -EINVAL;
3665 }
3666
3667 return 0;
f62b8bb8
AV		 3668}
3669
7d92d580
GP		 3670static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
3671 netdev_features_t features)
3672{
3673 struct mlx5e_priv *priv = netdev_priv(netdev);
6c3a823e 3674 struct mlx5e_params *params;
7d92d580
GP		 3675
3676 mutex_lock(&priv->state_lock);
6c3a823e 3677 params = &priv->channels.params;
7d92d580
GP		 3678 if (!bitmap_empty(priv->fs.vlan.active_svlans, VLAN_N_VID)) {
3679 /* HW strips the outer C-tag header, this is a problem
3680 * for S-tag traffic.
3681 */
3682 features &= ~NETIF_F_HW_VLAN_CTAG_RX;
6c3a823e 3683 if (!params->vlan_strip_disable)
7d92d580
GP		 3684 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n");
3685 }
6c3a823e
TT		 3686 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
3687 features &= ~NETIF_F_LRO;
3688 if (params->lro_en)
3689 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n");
3690 }
3691
7d92d580
GP		 3692 mutex_unlock(&priv->state_lock);
3693
3694 return features;
3695}
3696
250a42b6
AN		 3697int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
3698 change_hw_mtu_cb set_mtu_cb)
f62b8bb8
AV		 3699{
3700 struct mlx5e_priv *priv = netdev_priv(netdev);
2e20a151 3701 struct mlx5e_channels new_channels = {};
472a1e44 3702 struct mlx5e_params *params;
98e81b0a 3703 int err = 0;
506753b0 3704 bool reset;
f62b8bb8 3705
f62b8bb8 3706 mutex_lock(&priv->state_lock);
98e81b0a 3707
472a1e44 3708 params = &priv->channels.params;
506753b0 3709
73281b78 3710 reset = !params->lro_en;
2e20a151 3711 reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state);
98e81b0a 3712
73281b78
TT		 3713 new_channels.params = *params;
3714 new_channels.params.sw_mtu = new_mtu;
3715
a26a5bdf
TT		 3716 if (params->xdp_prog &&
3717 !mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
3718 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",
3719 new_mtu, MLX5E_XDP_MAX_MTU);
3720 err = -EINVAL;
3721 goto out;
3722 }
3723
99cbfa93 3724 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
73281b78
TT		 3725 u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params);
3726 u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params);
3727
3728 reset = reset && (ppw_old != ppw_new);
3729 }
3730
2e20a151 3731 if (!reset) {
472a1e44 3732 params->sw_mtu = new_mtu;
250a42b6 3733 set_mtu_cb(priv);
472a1e44 3734 netdev->mtu = params->sw_mtu;
2e20a151
SM		 3735 goto out;
3736 }
98e81b0a 3737
2e20a151 3738 err = mlx5e_open_channels(priv, &new_channels);
472a1e44 3739 if (err)
2e20a151 3740 goto out;
2e20a151 3741
250a42b6 3742 mlx5e_switch_priv_channels(priv, &new_channels, set_mtu_cb);
472a1e44 3743 netdev->mtu = new_channels.params.sw_mtu;
f62b8bb8 3744
2e20a151
SM		 3745out:
3746 mutex_unlock(&priv->state_lock);
f62b8bb8
AV		 3747 return err;
3748}
3749
250a42b6
AN		 3750static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu)
3751{
3752 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu);
3753}
3754
7c39afb3
FD		 3755int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr)
3756{
3757 struct hwtstamp_config config;
3758 int err;
3759
3760 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
3761 return -EOPNOTSUPP;
3762
3763 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
3764 return -EFAULT;
3765
3766 /* TX HW timestamp */
3767 switch (config.tx_type) {
3768 case HWTSTAMP_TX_OFF:
3769 case HWTSTAMP_TX_ON:
3770 break;
3771 default:
3772 return -ERANGE;
3773 }
3774
3775 mutex_lock(&priv->state_lock);
3776 /* RX HW timestamp */
3777 switch (config.rx_filter) {
3778 case HWTSTAMP_FILTER_NONE:
3779 /* Reset CQE compression to Admin default */
3780 mlx5e_modify_rx_cqe_compression_locked(priv, priv->channels.params.rx_cqe_compress_def);
3781 break;
3782 case HWTSTAMP_FILTER_ALL:
3783 case HWTSTAMP_FILTER_SOME:
3784 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
3785 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
3786 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
3787 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
3788 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
3789 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
3790 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
3791 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
3792 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
3793 case HWTSTAMP_FILTER_PTP_V2_EVENT:
3794 case HWTSTAMP_FILTER_PTP_V2_SYNC:
3795 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
3796 case HWTSTAMP_FILTER_NTP_ALL:
3797 /* Disable CQE compression */
3798 netdev_warn(priv->netdev, "Disabling cqe compression");
3799 err = mlx5e_modify_rx_cqe_compression_locked(priv, false);
3800 if (err) {
3801 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err);
3802 mutex_unlock(&priv->state_lock);
3803 return err;
3804 }
3805 config.rx_filter = HWTSTAMP_FILTER_ALL;
3806 break;
3807 default:
3808 mutex_unlock(&priv->state_lock);
3809 return -ERANGE;
3810 }
3811
3812 memcpy(&priv->tstamp, &config, sizeof(config));
3813 mutex_unlock(&priv->state_lock);
3814
3815 return copy_to_user(ifr->ifr_data, &config,
3816 sizeof(config)) ? -EFAULT : 0;
3817}
3818
3819int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr)
3820{
3821 struct hwtstamp_config *cfg = &priv->tstamp;
3822
3823 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
3824 return -EOPNOTSUPP;
3825
3826 return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0;
3827}
3828
ef9814de
EBE		 3829static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
3830{
1170fbd8
FD		 3831 struct mlx5e_priv *priv = netdev_priv(dev);
3832
ef9814de
EBE		 3833 switch (cmd) {
3834 case SIOCSHWTSTAMP:
1170fbd8 3835 return mlx5e_hwstamp_set(priv, ifr);
ef9814de 3836 case SIOCGHWTSTAMP:
1170fbd8 3837 return mlx5e_hwstamp_get(priv, ifr);
ef9814de
EBE		 3838 default:
3839 return -EOPNOTSUPP;
3840 }
3841}
3842
e80541ec 3843#ifdef CONFIG_MLX5_ESWITCH
66e49ded
SM		 3844static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
3845{
3846 struct mlx5e_priv *priv = netdev_priv(dev);
3847 struct mlx5_core_dev *mdev = priv->mdev;
3848
3849 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
3850}
3851
79aab093
MS		 3852static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
3853 __be16 vlan_proto)
66e49ded
SM		 3854{
3855 struct mlx5e_priv *priv = netdev_priv(dev);
3856 struct mlx5_core_dev *mdev = priv->mdev;
3857
79aab093
MS		 3858 if (vlan_proto != htons(ETH_P_8021Q))
3859 return -EPROTONOSUPPORT;
3860
66e49ded
SM		 3861 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
3862 vlan, qos);
3863}
3864
f942380c
MHY		 3865static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
3866{
3867 struct mlx5e_priv *priv = netdev_priv(dev);
3868 struct mlx5_core_dev *mdev = priv->mdev;
3869
3870 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
3871}
3872
1edc57e2
MHY		 3873static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
3874{
3875 struct mlx5e_priv *priv = netdev_priv(dev);
3876 struct mlx5_core_dev *mdev = priv->mdev;
3877
3878 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
3879}
bd77bf1c
MHY		 3880
3881static int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
3882 int max_tx_rate)
3883{
3884 struct mlx5e_priv *priv = netdev_priv(dev);
3885 struct mlx5_core_dev *mdev = priv->mdev;
3886
bd77bf1c 3887 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1,
c9497c98 3888 max_tx_rate, min_tx_rate);
bd77bf1c
MHY		 3889}
3890
66e49ded
SM		 3891static int mlx5_vport_link2ifla(u8 esw_link)
3892{
3893 switch (esw_link) {
3894 case MLX5_ESW_VPORT_ADMIN_STATE_DOWN:
3895 return IFLA_VF_LINK_STATE_DISABLE;
3896 case MLX5_ESW_VPORT_ADMIN_STATE_UP:
3897 return IFLA_VF_LINK_STATE_ENABLE;
3898 }
3899 return IFLA_VF_LINK_STATE_AUTO;
3900}
3901
3902static int mlx5_ifla_link2vport(u8 ifla_link)
3903{
3904 switch (ifla_link) {
3905 case IFLA_VF_LINK_STATE_DISABLE:
3906 return MLX5_ESW_VPORT_ADMIN_STATE_DOWN;
3907 case IFLA_VF_LINK_STATE_ENABLE:
3908 return MLX5_ESW_VPORT_ADMIN_STATE_UP;
3909 }
3910 return MLX5_ESW_VPORT_ADMIN_STATE_AUTO;
3911}
3912
3913static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
3914 int link_state)
3915{
3916 struct mlx5e_priv *priv = netdev_priv(dev);
3917 struct mlx5_core_dev *mdev = priv->mdev;
3918
3919 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
3920 mlx5_ifla_link2vport(link_state));
3921}
3922
3923static int mlx5e_get_vf_config(struct net_device *dev,
3924 int vf, struct ifla_vf_info *ivi)
3925{
3926 struct mlx5e_priv *priv = netdev_priv(dev);
3927 struct mlx5_core_dev *mdev = priv->mdev;
3928 int err;
3929
3930 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
3931 if (err)
3932 return err;
3933 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
3934 return 0;
3935}
3936
3937static int mlx5e_get_vf_stats(struct net_device *dev,
3938 int vf, struct ifla_vf_stats *vf_stats)
3939{
3940 struct mlx5e_priv *priv = netdev_priv(dev);
3941 struct mlx5_core_dev *mdev = priv->mdev;
3942
3943 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
3944 vf_stats);
3945}
e80541ec 3946#endif
66e49ded 3947
1ad9a00a
PB		 3948static void mlx5e_add_vxlan_port(struct net_device *netdev,
3949 struct udp_tunnel_info *ti)
b3f63c3d
MF		 3950{
3951 struct mlx5e_priv *priv = netdev_priv(netdev);
3952
974c3f30
AD		 3953 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3954 return;
3955
b3f63c3d
MF		 3956 if (!mlx5e_vxlan_allowed(priv->mdev))
3957 return;
3958
974c3f30 3959 mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 1);
b3f63c3d
MF		 3960}
3961
1ad9a00a
PB		 3962static void mlx5e_del_vxlan_port(struct net_device *netdev,
3963 struct udp_tunnel_info *ti)
b3f63c3d
MF		 3964{
3965 struct mlx5e_priv *priv = netdev_priv(netdev);
3966
974c3f30
AD		 3967 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3968 return;
3969
b3f63c3d
MF		 3970 if (!mlx5e_vxlan_allowed(priv->mdev))
3971 return;
3972
974c3f30 3973 mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
b3f63c3d
MF		 3974}
3975
27299841
GP		 3976static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
3977 struct sk_buff *skb,
3978 netdev_features_t features)
b3f63c3d 3979{
2989ad1e 3980 unsigned int offset = 0;
b3f63c3d 3981 struct udphdr *udph;
27299841
GP		 3982 u8 proto;
3983 u16 port;
b3f63c3d
MF		 3984
3985 switch (vlan_get_protocol(skb)) {
3986 case htons(ETH_P_IP):
3987 proto = ip_hdr(skb)->protocol;
3988 break;
3989 case htons(ETH_P_IPV6):
2989ad1e 3990 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL);
b3f63c3d
MF		 3991 break;
3992 default:
3993 goto out;
3994 }
3995
27299841
GP		 3996 switch (proto) {
3997 case IPPROTO_GRE:
3998 return features;
3999 case IPPROTO_UDP:
b3f63c3d
MF		 4000 udph = udp_hdr(skb);
4001 port = be16_to_cpu(udph->dest);
b3f63c3d 4002
27299841
GP		 4003 /* Verify if UDP port is being offloaded by HW */
4004 if (mlx5e_vxlan_lookup_port(priv, port))
4005 return features;
4006 }
b3f63c3d
MF		 4007
4008out:
4009 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
4010 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4011}
4012
4013static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
4014 struct net_device *netdev,
4015 netdev_features_t features)
4016{
4017 struct mlx5e_priv *priv = netdev_priv(netdev);
4018
4019 features = vlan_features_check(skb, features);
4020 features = vxlan_features_check(skb, features);
4021
2ac9cfe7
IT		 4022#ifdef CONFIG_MLX5_EN_IPSEC
4023 if (mlx5e_ipsec_feature_check(skb, netdev, features))
4024 return features;
4025#endif
4026
b3f63c3d
MF		 4027 /* Validate if the tunneled packet is being offloaded by HW */
4028 if (skb->encapsulation &&
4029 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
27299841 4030 return mlx5e_tunnel_features_check(priv, skb, features);
b3f63c3d
MF		 4031
4032 return features;
4033}
4034
7ca560b5
EBE		 4035static bool mlx5e_tx_timeout_eq_recover(struct net_device *dev,
4036 struct mlx5e_txqsq *sq)
4037{
7b2117bb 4038 struct mlx5_eq *eq = sq->cq.mcq.eq;
7ca560b5
EBE		 4039 u32 eqe_count;
4040
7ca560b5 4041 netdev_err(dev, "EQ 0x%x: Cons = 0x%x, irqn = 0x%x\n",
7b2117bb 4042 eq->eqn, eq->cons_index, eq->irqn);
7ca560b5
EBE		 4043
4044 eqe_count = mlx5_eq_poll_irq_disabled(eq);
4045 if (!eqe_count)
4046 return false;
4047
4048 netdev_err(dev, "Recover %d eqes on EQ 0x%x\n", eqe_count, eq->eqn);
05909bab 4049 sq->channel->stats->eq_rearm++;
7ca560b5
EBE		 4050 return true;
4051}
4052
bfc647d5 4053static void mlx5e_tx_timeout_work(struct work_struct *work)
3947ca18 4054{
bfc647d5
EBE		 4055 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
4056 tx_timeout_work);
4057 struct net_device *dev = priv->netdev;
7ca560b5 4058 bool reopen_channels = false;
bfc647d5 4059 int i, err;
3947ca18 4060
bfc647d5
EBE		 4061 rtnl_lock();
4062 mutex_lock(&priv->state_lock);
4063
4064 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
4065 goto unlock;
3947ca18 4066
6a9764ef 4067 for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) {
84990945 4068 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, i);
acc6c595 4069 struct mlx5e_txqsq *sq = priv->txq2sq[i];
3947ca18 4070
84990945 4071 if (!netif_xmit_stopped(dev_queue))
3947ca18 4072 continue;
bfc647d5
EBE		 4073
4074 netdev_err(dev,
4075 "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u\n",
84990945
EBE		 4076 i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc,
4077 jiffies_to_usecs(jiffies - dev_queue->trans_start));
3a32b26a 4078
7ca560b5
EBE		 4079 /* If we recover a lost interrupt, most likely TX timeout will
4080 * be resolved, skip reopening channels
4081 */
4082 if (!mlx5e_tx_timeout_eq_recover(dev, sq)) {
4083 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
4084 reopen_channels = true;
4085 }
3947ca18
DJ		 4086 }
4087
bfc647d5
EBE		 4088 if (!reopen_channels)
4089 goto unlock;
4090
4091 mlx5e_close_locked(dev);
4092 err = mlx5e_open_locked(dev);
4093 if (err)
4094 netdev_err(priv->netdev,
4095 "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n",
4096 err);
4097
4098unlock:
4099 mutex_unlock(&priv->state_lock);
4100 rtnl_unlock();
4101}
4102
4103static void mlx5e_tx_timeout(struct net_device *dev)
4104{
4105 struct mlx5e_priv *priv = netdev_priv(dev);
4106
4107 netdev_err(dev, "TX timeout detected\n");
4108 queue_work(priv->wq, &priv->tx_timeout_work);
3947ca18
DJ		 4109}
4110
a26a5bdf 4111static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog)
0ec13877
TT		 4112{
4113 struct net_device *netdev = priv->netdev;
a26a5bdf 4114 struct mlx5e_channels new_channels = {};
0ec13877
TT		 4115
4116 if (priv->channels.params.lro_en) {
4117 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
4118 return -EINVAL;
4119 }
4120
4121 if (MLX5_IPSEC_DEV(priv->mdev)) {
4122 netdev_warn(netdev, "can't set XDP with IPSec offload\n");
4123 return -EINVAL;
4124 }
4125
a26a5bdf
TT		 4126 new_channels.params = priv->channels.params;
4127 new_channels.params.xdp_prog = prog;
4128
4129 if (!mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
4130 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",
4131 new_channels.params.sw_mtu, MLX5E_XDP_MAX_MTU);
4132 return -EINVAL;
4133 }
4134
0ec13877
TT		 4135 return 0;
4136}
4137
86994156
RS		 4138static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
4139{
4140 struct mlx5e_priv *priv = netdev_priv(netdev);
4141 struct bpf_prog *old_prog;
86994156 4142 bool reset, was_opened;
0ec13877 4143 int err;
86994156
RS		 4144 int i;
4145
4146 mutex_lock(&priv->state_lock);
4147
0ec13877 4148 if (prog) {
a26a5bdf 4149 err = mlx5e_xdp_allowed(priv, prog);
0ec13877
TT		 4150 if (err)
4151 goto unlock;
547eede0
IT		 4152 }
4153
86994156
RS		 4154 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
4155 /* no need for full reset when exchanging programs */
6a9764ef 4156 reset = (!priv->channels.params.xdp_prog || !prog);
86994156
RS		 4157
4158 if (was_opened && reset)
4159 mlx5e_close_locked(netdev);
c54c0629
DB		 4160 if (was_opened && !reset) {
4161 /* num_channels is invariant here, so we can take the
4162 * batched reference right upfront.
4163 */
6a9764ef 4164 prog = bpf_prog_add(prog, priv->channels.num);
c54c0629
DB		 4165 if (IS_ERR(prog)) {
4166 err = PTR_ERR(prog);
4167 goto unlock;
4168 }
4169 }
86994156 4170
c54c0629
DB		 4171 /* exchange programs, extra prog reference we got from caller
4172 * as long as we don't fail from this point onwards.
4173 */
6a9764ef 4174 old_prog = xchg(&priv->channels.params.xdp_prog, prog);
86994156
RS		 4175 if (old_prog)
4176 bpf_prog_put(old_prog);
4177
4178 if (reset) /* change RQ type according to priv->xdp_prog */
2a0f561b 4179 mlx5e_set_rq_type(priv->mdev, &priv->channels.params);
86994156
RS		 4180
4181 if (was_opened && reset)
4182 mlx5e_open_locked(netdev);
4183
4184 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
4185 goto unlock;
4186
4187 /* exchanging programs w/o reset, we update ref counts on behalf
4188 * of the channels RQs here.
4189 */
ff9c852f
SM		 4190 for (i = 0; i < priv->channels.num; i++) {
4191 struct mlx5e_channel *c = priv->channels.c[i];
86994156 4192
c0f1147d 4193 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
86994156
RS		 4194 napi_synchronize(&c->napi);
4195 /* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */
4196
4197 old_prog = xchg(&c->rq.xdp_prog, prog);
4198
c0f1147d 4199 set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
86994156 4200 /* napi_schedule in case we have missed anything */
86994156
RS		 4201 napi_schedule(&c->napi);
4202
4203 if (old_prog)
4204 bpf_prog_put(old_prog);
4205 }
4206
4207unlock:
4208 mutex_unlock(&priv->state_lock);
4209 return err;
4210}
4211
821b2e29 4212static u32 mlx5e_xdp_query(struct net_device *dev)
86994156
RS		 4213{
4214 struct mlx5e_priv *priv = netdev_priv(dev);
821b2e29
MKL		 4215 const struct bpf_prog *xdp_prog;
4216 u32 prog_id = 0;
86994156 4217
821b2e29
MKL		 4218 mutex_lock(&priv->state_lock);
4219 xdp_prog = priv->channels.params.xdp_prog;
4220 if (xdp_prog)
4221 prog_id = xdp_prog->aux->id;
4222 mutex_unlock(&priv->state_lock);
4223
4224 return prog_id;
86994156
RS		 4225}
4226
f4e63525 4227static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp)
86994156
RS		 4228{
4229 switch (xdp->command) {
4230 case XDP_SETUP_PROG:
4231 return mlx5e_xdp_set(dev, xdp->prog);
4232 case XDP_QUERY_PROG:
821b2e29 4233 xdp->prog_id = mlx5e_xdp_query(dev);
86994156
RS		 4234 return 0;
4235 default:
4236 return -EINVAL;
4237 }
4238}
4239
80378384
CO		 4240#ifdef CONFIG_NET_POLL_CONTROLLER
4241/* Fake "interrupt" called by netpoll (eg netconsole) to send skbs without
4242 * reenabling interrupts.
4243 */
4244static void mlx5e_netpoll(struct net_device *dev)
4245{
4246 struct mlx5e_priv *priv = netdev_priv(dev);
ff9c852f
SM		 4247 struct mlx5e_channels *chs = &priv->channels;
4248
80378384
CO		 4249 int i;
4250
ff9c852f
SM		 4251 for (i = 0; i < chs->num; i++)
4252 napi_schedule(&chs->c[i]->napi);
80378384
CO		 4253}
4254#endif
4255
e80541ec 4256static const struct net_device_ops mlx5e_netdev_ops = {
f62b8bb8
AV		 4257 .ndo_open = mlx5e_open,
4258 .ndo_stop = mlx5e_close,
4259 .ndo_start_xmit = mlx5e_xmit,
0cf0f6d3 4260 .ndo_setup_tc = mlx5e_setup_tc,
08fb1dac 4261 .ndo_select_queue = mlx5e_select_queue,
f62b8bb8
AV		 4262 .ndo_get_stats64 = mlx5e_get_stats,
4263 .ndo_set_rx_mode = mlx5e_set_rx_mode,
4264 .ndo_set_mac_address = mlx5e_set_mac,
b0eed40e
SM		 4265 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid,
4266 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid,
f62b8bb8 4267 .ndo_set_features = mlx5e_set_features,
7d92d580 4268 .ndo_fix_features = mlx5e_fix_features,
250a42b6 4269 .ndo_change_mtu = mlx5e_change_nic_mtu,
b0eed40e 4270 .ndo_do_ioctl = mlx5e_ioctl,
507f0c81 4271 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate,
706b3583
SM		 4272 .ndo_udp_tunnel_add = mlx5e_add_vxlan_port,
4273 .ndo_udp_tunnel_del = mlx5e_del_vxlan_port,
4274 .ndo_features_check = mlx5e_features_check,
45bf454a
MG		 4275#ifdef CONFIG_RFS_ACCEL
4276 .ndo_rx_flow_steer = mlx5e_rx_flow_steer,
4277#endif
3947ca18 4278 .ndo_tx_timeout = mlx5e_tx_timeout,
f4e63525 4279 .ndo_bpf = mlx5e_xdp,
80378384
CO		 4280#ifdef CONFIG_NET_POLL_CONTROLLER
4281 .ndo_poll_controller = mlx5e_netpoll,
4282#endif
e80541ec 4283#ifdef CONFIG_MLX5_ESWITCH
706b3583 4284 /* SRIOV E-Switch NDOs */
b0eed40e
SM		 4285 .ndo_set_vf_mac = mlx5e_set_vf_mac,
4286 .ndo_set_vf_vlan = mlx5e_set_vf_vlan,
f942380c 4287 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk,
1edc57e2 4288 .ndo_set_vf_trust = mlx5e_set_vf_trust,
bd77bf1c 4289 .ndo_set_vf_rate = mlx5e_set_vf_rate,
b0eed40e
SM		 4290 .ndo_get_vf_config = mlx5e_get_vf_config,
4291 .ndo_set_vf_link_state = mlx5e_set_vf_link_state,
4292 .ndo_get_vf_stats = mlx5e_get_vf_stats,
370bad0f
OG		 4293 .ndo_has_offload_stats = mlx5e_has_offload_stats,
4294 .ndo_get_offload_stats = mlx5e_get_offload_stats,
e80541ec 4295#endif
f62b8bb8
AV		 4296};
4297
4298static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
4299{
4300 if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
9eb78923 4301 return -EOPNOTSUPP;
f62b8bb8
AV		 4302 if (!MLX5_CAP_GEN(mdev, eth_net_offloads) ||
4303 !MLX5_CAP_GEN(mdev, nic_flow_table) ||
4304 !MLX5_CAP_ETH(mdev, csum_cap) ||
4305 !MLX5_CAP_ETH(mdev, max_lso_cap) ||
4306 !MLX5_CAP_ETH(mdev, vlan_cap) ||
796a27ec
GP		 4307 !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) ||
4308 MLX5_CAP_FLOWTABLE(mdev,
4309 flow_table_properties_nic_receive.max_ft_level)
4310 < 3) {
f62b8bb8
AV		 4311 mlx5_core_warn(mdev,
4312 "Not creating net device, some required device capabilities are missing\n");
9eb78923 4313 return -EOPNOTSUPP;
f62b8bb8 4314 }
66189961
TT		 4315 if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable))
4316 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n");
7524a5d8 4317 if (!MLX5_CAP_GEN(mdev, cq_moderation))
3e432ab6 4318 mlx5_core_warn(mdev, "CQ moderation is not supported\n");
66189961 4319
f62b8bb8
AV		 4320 return 0;
4321}
4322
d4b6c488 4323void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
85082dba
TT		 4324 int num_channels)
4325{
4326 int i;
4327
4328 for (i = 0; i < len; i++)
4329 indirection_rqt[i] = i % num_channels;
4330}
4331
0608d4db 4332static bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
b797a684 4333{
0608d4db
TT		 4334 u32 link_speed = 0;
4335 u32 pci_bw = 0;
b797a684 4336
2c81bfd5 4337 mlx5e_port_max_linkspeed(mdev, &link_speed);
3c0d551e 4338 pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL);
0608d4db
TT		 4339 mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n",
4340 link_speed, pci_bw);
4341
4342#define MLX5E_SLOW_PCI_RATIO (2)
4343
4344 return link_speed && pci_bw &&
4345 link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw;
0f6e4cf6
EBE		 4346}
4347
cbce4f44 4348static struct net_dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode)
0088cbbc 4349{
cbce4f44
TG		 4350 struct net_dim_cq_moder moder;
4351
4352 moder.cq_period_mode = cq_period_mode;
4353 moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
4354 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
4355 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
4356 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE;
4357
4358 return moder;
4359}
0088cbbc 4360
cbce4f44
TG		 4361static struct net_dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode)
4362{
4363 struct net_dim_cq_moder moder;
0088cbbc 4364
cbce4f44
TG		 4365 moder.cq_period_mode = cq_period_mode;
4366 moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
4367 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
0088cbbc 4368 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
cbce4f44
TG		 4369 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
4370
4371 return moder;
4372}
4373
4374static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode)
4375{
4376 return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ?
4377 NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE :
4378 NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
4379}
4380
4381void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
4382{
4383 if (params->tx_dim_enabled) {
4384 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
4385
4386 params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode);
4387 } else {
4388 params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode);
4389 }
0088cbbc
TG		 4390
4391 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER,
4392 params->tx_cq_moderation.cq_period_mode ==
4393 MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
4394}
4395
9908aa29
TT		 4396void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
4397{
9a317425 4398 if (params->rx_dim_enabled) {
cbce4f44
TG		 4399 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
4400
4401 params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode);
4402 } else {
4403 params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode);
9a317425 4404 }
457fcd8a 4405
6a9764ef 4406 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
0088cbbc
TG		 4407 params->rx_cq_moderation.cq_period_mode ==
4408 MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
9908aa29
TT		 4409}
4410
707129dc 4411static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
2b029556
SM		 4412{
4413 int i;
4414
4415 /* The supported periods are organized in ascending order */
4416 for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++)
4417 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout)
4418 break;
4419
4420 return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
4421}
4422
8f493ffd
SM		 4423void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
4424 struct mlx5e_params *params,
472a1e44 4425 u16 max_channels, u16 mtu)
f62b8bb8 4426{
48bfc397 4427 u8 rx_cq_period_mode;
2fc4bfb7 4428
472a1e44
TT		 4429 params->sw_mtu = mtu;
4430 params->hard_mtu = MLX5E_ETH_HARD_MTU;
6a9764ef
SM		 4431 params->num_channels = max_channels;
4432 params->num_tc = 1;
2b029556 4433
6a9764ef
SM		 4434 /* SQ */
4435 params->log_sq_size = is_kdump_kernel() ?
b4e029da
KH		 4436 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
4437 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
461017cb 4438
b797a684 4439 /* set CQE compression */
6a9764ef 4440 params->rx_cqe_compress_def = false;
b797a684 4441 if (MLX5_CAP_GEN(mdev, cqe_compression) &&
e53eef63 4442 MLX5_CAP_GEN(mdev, vport_group_manager))
0608d4db 4443 params->rx_cqe_compress_def = slow_pci_heuristic(mdev);
0f6e4cf6 4444
6a9764ef
SM		 4445 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def);
4446
4447 /* RQ */
5ffd8194
TT		 4448 /* Prefer Striding RQ, unless any of the following holds:
4449 * - Striding RQ configuration is not possible/supported.
4450 * - Slow PCI heuristic.
4451 * - Legacy RQ would use linear SKB while Striding RQ would use non-linear.
4452 */
4453 if (!slow_pci_heuristic(mdev) &&
4454 mlx5e_striding_rq_possible(mdev, params) &&
4455 (mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ||
4456 !mlx5e_rx_is_linear_skb(mdev, params)))
4457 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
2a0f561b
TT		 4458 mlx5e_set_rq_type(mdev, params);
4459 mlx5e_init_rq_type_params(mdev, params);
b797a684 4460
6a9764ef 4461 /* HW LRO */
c139dbfd 4462
5426a0b2 4463 /* TODO: && MLX5_CAP_ETH(mdev, lro_cap) */
6a9764ef 4464 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
619a8f2a
TT		 4465 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
4466 params->lro_en = !slow_pci_heuristic(mdev);
6a9764ef 4467 params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
b0d4660b 4468
6a9764ef 4469 /* CQ moderation params */
48bfc397 4470 rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
6a9764ef
SM		 4471 MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
4472 MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
9a317425 4473 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
cbce4f44 4474 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
48bfc397
TG		 4475 mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode);
4476 mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
9908aa29 4477
6a9764ef 4478 /* TX inline */
fbcb127e 4479 params->tx_min_inline_mode = mlx5e_params_calculate_tx_min_inline(mdev);
a6f402e4 4480
6a9764ef
SM		 4481 /* RSS */
4482 params->rss_hfunc = ETH_RSS_HASH_XOR;
4483 netdev_rss_key_fill(params->toeplitz_hash_key, sizeof(params->toeplitz_hash_key));
d4b6c488 4484 mlx5e_build_default_indir_rqt(params->indirection_rqt,
6a9764ef
SM		 4485 MLX5E_INDIR_RQT_SIZE, max_channels);
4486}
f62b8bb8 4487
6a9764ef
SM		 4488static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
4489 struct net_device *netdev,
4490 const struct mlx5e_profile *profile,
4491 void *ppriv)
4492{
4493 struct mlx5e_priv *priv = netdev_priv(netdev);
57afead5 4494
6a9764ef
SM		 4495 priv->mdev = mdev;
4496 priv->netdev = netdev;
4497 priv->profile = profile;
4498 priv->ppriv = ppriv;
79c48764 4499 priv->msglevel = MLX5E_MSG_LEVEL;
05909bab 4500 priv->max_opened_tc = 1;
2d75b2bc 4501
472a1e44
TT		 4502 mlx5e_build_nic_params(mdev, &priv->channels.params,
4503 profile->max_nch(mdev), netdev->mtu);
9908aa29 4504
f62b8bb8
AV		 4505 mutex_init(&priv->state_lock);
4506
4507 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
4508 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3947ca18 4509 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
f62b8bb8 4510 INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
237f258c
FD		 4511
4512 mlx5e_timestamp_init(priv);
f62b8bb8
AV		 4513}
4514
4515static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
4516{
4517 struct mlx5e_priv *priv = netdev_priv(netdev);
4518
e1d7d349 4519 mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr);
108805fc
SM		 4520 if (is_zero_ether_addr(netdev->dev_addr) &&
4521 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
4522 eth_hw_addr_random(netdev);
4523 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
4524 }
f62b8bb8
AV		 4525}
4526
f125376b 4527#if IS_ENABLED(CONFIG_MLX5_ESWITCH)
cb67b832
HHZ		 4528static const struct switchdev_ops mlx5e_switchdev_ops = {
4529 .switchdev_port_attr_get = mlx5e_attr_get,
4530};
e80541ec 4531#endif
cb67b832 4532
6bfd390b 4533static void mlx5e_build_nic_netdev(struct net_device *netdev)
f62b8bb8
AV		 4534{
4535 struct mlx5e_priv *priv = netdev_priv(netdev);
4536 struct mlx5_core_dev *mdev = priv->mdev;
94cb1ebb
EBE		 4537 bool fcs_supported;
4538 bool fcs_enabled;
f62b8bb8
AV		 4539
4540 SET_NETDEV_DEV(netdev, &mdev->pdev->dev);
4541
e80541ec
SM		 4542 netdev->netdev_ops = &mlx5e_netdev_ops;
4543
08fb1dac 4544#ifdef CONFIG_MLX5_CORE_EN_DCB
e80541ec
SM		 4545 if (MLX5_CAP_GEN(mdev, vport_group_manager) && MLX5_CAP_GEN(mdev, qos))
4546 netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
08fb1dac 4547#endif
66e49ded 4548
f62b8bb8
AV		 4549 netdev->watchdog_timeo = 15 * HZ;
4550
4551 netdev->ethtool_ops = &mlx5e_ethtool_ops;
4552
12be4b21 4553 netdev->vlan_features |= NETIF_F_SG;
f62b8bb8
AV		 4554 netdev->vlan_features |= NETIF_F_IP_CSUM;
4555 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
4556 netdev->vlan_features |= NETIF_F_GRO;
4557 netdev->vlan_features |= NETIF_F_TSO;
4558 netdev->vlan_features |= NETIF_F_TSO6;
4559 netdev->vlan_features |= NETIF_F_RXCSUM;
4560 netdev->vlan_features |= NETIF_F_RXHASH;
4561
71186172
AH		 4562 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX;
4563 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX;
4564
6c3a823e
TT		 4565 if (!!MLX5_CAP_ETH(mdev, lro_cap) &&
4566 mlx5e_check_fragmented_striding_rq_cap(mdev))
f62b8bb8
AV		 4567 netdev->vlan_features |= NETIF_F_LRO;
4568
4569 netdev->hw_features = netdev->vlan_features;
e4cf27bd 4570 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
f62b8bb8
AV		 4571 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4572 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4382c7b9 4573 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX;
f62b8bb8 4574
27299841 4575 if (mlx5e_vxlan_allowed(mdev) || MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) {
b3f63c3d 4576 netdev->hw_enc_features |= NETIF_F_IP_CSUM;
f3ed653c 4577 netdev->hw_enc_features |= NETIF_F_IPV6_CSUM;
b3f63c3d
MF		 4578 netdev->hw_enc_features |= NETIF_F_TSO;
4579 netdev->hw_enc_features |= NETIF_F_TSO6;
27299841
GP		 4580 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL;
4581 }
4582
4583 if (mlx5e_vxlan_allowed(mdev)) {
4584 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
4585 NETIF_F_GSO_UDP_TUNNEL_CSUM;
4586 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
4587 NETIF_F_GSO_UDP_TUNNEL_CSUM;
b49663c8 4588 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
b3f63c3d
MF		 4589 }
4590
27299841
GP		 4591 if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) {
4592 netdev->hw_features |= NETIF_F_GSO_GRE |
4593 NETIF_F_GSO_GRE_CSUM;
4594 netdev->hw_enc_features |= NETIF_F_GSO_GRE |
4595 NETIF_F_GSO_GRE_CSUM;
4596 netdev->gso_partial_features |= NETIF_F_GSO_GRE |
4597 NETIF_F_GSO_GRE_CSUM;
4598 }
4599
3f44899e
BP		 4600 netdev->hw_features |= NETIF_F_GSO_PARTIAL;
4601 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4;
4602 netdev->hw_features |= NETIF_F_GSO_UDP_L4;
4603 netdev->features |= NETIF_F_GSO_UDP_L4;
4604
94cb1ebb
EBE		 4605 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
4606
4607 if (fcs_supported)
4608 netdev->hw_features |= NETIF_F_RXALL;
4609
102722fc
GE		 4610 if (MLX5_CAP_ETH(mdev, scatter_fcs))
4611 netdev->hw_features |= NETIF_F_RXFCS;
4612
f62b8bb8 4613 netdev->features = netdev->hw_features;
6a9764ef 4614 if (!priv->channels.params.lro_en)
f62b8bb8
AV		 4615 netdev->features &= ~NETIF_F_LRO;
4616
94cb1ebb
EBE		 4617 if (fcs_enabled)
4618 netdev->features &= ~NETIF_F_RXALL;
4619
102722fc
GE		 4620 if (!priv->channels.params.scatter_fcs_en)
4621 netdev->features &= ~NETIF_F_RXFCS;
4622
e8f887ac
AV		 4623#define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
4624 if (FT_CAP(flow_modify_en) &&
4625 FT_CAP(modify_root) &&
4626 FT_CAP(identified_miss_table_mode) &&
1cabe6b0
MG		 4627 FT_CAP(flow_table_modify)) {
4628 netdev->hw_features |= NETIF_F_HW_TC;
4629#ifdef CONFIG_RFS_ACCEL
4630 netdev->hw_features |= NETIF_F_NTUPLE;
4631#endif
4632 }
e8f887ac 4633
f62b8bb8 4634 netdev->features |= NETIF_F_HIGHDMA;
7d92d580 4635 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
f62b8bb8
AV		 4636
4637 netdev->priv_flags |= IFF_UNICAST_FLT;
4638
4639 mlx5e_set_netdev_dev_addr(netdev);
cb67b832 4640
f125376b 4641#if IS_ENABLED(CONFIG_MLX5_ESWITCH)
733d3e54 4642 if (MLX5_ESWITCH_MANAGER(mdev))
cb67b832
HHZ		 4643 netdev->switchdev_ops = &mlx5e_switchdev_ops;
4644#endif
547eede0
IT		 4645
4646 mlx5e_ipsec_build_netdev(priv);
c83294b9 4647 mlx5e_tls_build_netdev(priv);
f62b8bb8
AV		 4648}
4649
7cbaf9a3 4650static void mlx5e_create_q_counters(struct mlx5e_priv *priv)
593cf338
RS		 4651{
4652 struct mlx5_core_dev *mdev = priv->mdev;
4653 int err;
4654
4655 err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter);
4656 if (err) {
4657 mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err);
4658 priv->q_counter = 0;
4659 }
7cbaf9a3
MS		 4660
4661 err = mlx5_core_alloc_q_counter(mdev, &priv->drop_rq_q_counter);
4662 if (err) {
4663 mlx5_core_warn(mdev, "alloc drop RQ counter failed, %d\n", err);
4664 priv->drop_rq_q_counter = 0;
4665 }
593cf338
RS		 4666}
4667
7cbaf9a3 4668static void mlx5e_destroy_q_counters(struct mlx5e_priv *priv)
593cf338 4669{
7cbaf9a3
MS		 4670 if (priv->q_counter)
4671 mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter);
593cf338 4672
7cbaf9a3
MS		 4673 if (priv->drop_rq_q_counter)
4674 mlx5_core_dealloc_q_counter(priv->mdev, priv->drop_rq_q_counter);
593cf338
RS		 4675}
4676
6bfd390b
HHZ		 4677static void mlx5e_nic_init(struct mlx5_core_dev *mdev,
4678 struct net_device *netdev,
127ea380
HHZ		 4679 const struct mlx5e_profile *profile,
4680 void *ppriv)
6bfd390b
HHZ		 4681{
4682 struct mlx5e_priv *priv = netdev_priv(netdev);
547eede0 4683 int err;
6bfd390b 4684
127ea380 4685 mlx5e_build_nic_netdev_priv(mdev, netdev, profile, ppriv);
547eede0
IT		 4686 err = mlx5e_ipsec_init(priv);
4687 if (err)
4688 mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err);
43585a41
IL		 4689 err = mlx5e_tls_init(priv);
4690 if (err)
4691 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err);
6bfd390b 4692 mlx5e_build_nic_netdev(netdev);
8bfaf07f 4693 mlx5e_build_tc2txq_maps(priv);
6bfd390b
HHZ		 4694 mlx5e_vxlan_init(priv);
4695}
4696
4697static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
4698{
43585a41 4699 mlx5e_tls_cleanup(priv);
547eede0 4700 mlx5e_ipsec_cleanup(priv);
6bfd390b
HHZ		 4701 mlx5e_vxlan_cleanup(priv);
4702}
4703
4704static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
4705{
4706 struct mlx5_core_dev *mdev = priv->mdev;
4707 int err;
6bfd390b 4708
8f493ffd
SM		 4709 err = mlx5e_create_indirect_rqt(priv);
4710 if (err)
6bfd390b 4711 return err;
6bfd390b
HHZ		 4712
4713 err = mlx5e_create_direct_rqts(priv);
8f493ffd 4714 if (err)
6bfd390b 4715 goto err_destroy_indirect_rqts;
6bfd390b
HHZ		 4716
4717 err = mlx5e_create_indirect_tirs(priv);
8f493ffd 4718 if (err)
6bfd390b 4719 goto err_destroy_direct_rqts;
6bfd390b
HHZ		 4720
4721 err = mlx5e_create_direct_tirs(priv);
8f493ffd 4722 if (err)
6bfd390b 4723 goto err_destroy_indirect_tirs;
6bfd390b
HHZ		 4724
4725 err = mlx5e_create_flow_steering(priv);
4726 if (err) {
4727 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
4728 goto err_destroy_direct_tirs;
4729 }
4730
655dc3d2 4731 err = mlx5e_tc_nic_init(priv);
6bfd390b
HHZ		 4732 if (err)
4733 goto err_destroy_flow_steering;
4734
4735 return 0;
4736
4737err_destroy_flow_steering:
4738 mlx5e_destroy_flow_steering(priv);
4739err_destroy_direct_tirs:
4740 mlx5e_destroy_direct_tirs(priv);
4741err_destroy_indirect_tirs:
4742 mlx5e_destroy_indirect_tirs(priv);
4743err_destroy_direct_rqts:
8f493ffd 4744 mlx5e_destroy_direct_rqts(priv);
6bfd390b
HHZ		 4745err_destroy_indirect_rqts:
4746 mlx5e_destroy_rqt(priv, &priv->indir_rqt);
4747 return err;
4748}
4749
4750static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
4751{
655dc3d2 4752 mlx5e_tc_nic_cleanup(priv);
6bfd390b
HHZ		 4753 mlx5e_destroy_flow_steering(priv);
4754 mlx5e_destroy_direct_tirs(priv);
4755 mlx5e_destroy_indirect_tirs(priv);
8f493ffd 4756 mlx5e_destroy_direct_rqts(priv);
6bfd390b
HHZ		 4757 mlx5e_destroy_rqt(priv, &priv->indir_rqt);
4758}
4759
4760static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
4761{
4762 int err;
4763
4764 err = mlx5e_create_tises(priv);
4765 if (err) {
4766 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
4767 return err;
4768 }
4769
4770#ifdef CONFIG_MLX5_CORE_EN_DCB
e207b7e9 4771 mlx5e_dcbnl_initialize(priv);
6bfd390b
HHZ		 4772#endif
4773 return 0;
4774}
4775
4776static void mlx5e_nic_enable(struct mlx5e_priv *priv)
4777{
4778 struct net_device *netdev = priv->netdev;
4779 struct mlx5_core_dev *mdev = priv->mdev;
2c3b5bee
SM		 4780 u16 max_mtu;
4781
4782 mlx5e_init_l2_addr(priv);
4783
63bfd399
EBE		 4784 /* Marking the link as currently not needed by the Driver */
4785 if (!netif_running(netdev))
4786 mlx5_set_port_admin_status(mdev, MLX5_PORT_DOWN);
4787
2c3b5bee
SM		 4788 /* MTU range: 68 - hw-specific max */
4789 netdev->min_mtu = ETH_MIN_MTU;
4790 mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1);
472a1e44 4791 netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu);
2c3b5bee 4792 mlx5e_set_dev_port_mtu(priv);
6bfd390b 4793
7907f23a
AH		 4794 mlx5_lag_add(mdev, netdev);
4795
6bfd390b 4796 mlx5e_enable_async_events(priv);
127ea380 4797
733d3e54 4798 if (MLX5_ESWITCH_MANAGER(priv->mdev))
1d447a39 4799 mlx5e_register_vport_reps(priv);
2c3b5bee 4800
610e89e0
SM		 4801 if (netdev->reg_state != NETREG_REGISTERED)
4802 return;
2a5e7a13
HN		 4803#ifdef CONFIG_MLX5_CORE_EN_DCB
4804 mlx5e_dcbnl_init_app(priv);
4805#endif
610e89e0
SM		 4806
4807 queue_work(priv->wq, &priv->set_rx_mode_work);
2c3b5bee
SM		 4808
4809 rtnl_lock();
4810 if (netif_running(netdev))
4811 mlx5e_open(netdev);
4812 netif_device_attach(netdev);
4813 rtnl_unlock();
6bfd390b
HHZ		 4814}
4815
4816static void mlx5e_nic_disable(struct mlx5e_priv *priv)
4817{
3deef8ce 4818 struct mlx5_core_dev *mdev = priv->mdev;
3deef8ce 4819
2a5e7a13
HN		 4820#ifdef CONFIG_MLX5_CORE_EN_DCB
4821 if (priv->netdev->reg_state == NETREG_REGISTERED)
4822 mlx5e_dcbnl_delete_app(priv);
4823#endif
4824
2c3b5bee
SM		 4825 rtnl_lock();
4826 if (netif_running(priv->netdev))
4827 mlx5e_close(priv->netdev);
4828 netif_device_detach(priv->netdev);
4829 rtnl_unlock();
4830
6bfd390b 4831 queue_work(priv->wq, &priv->set_rx_mode_work);
1d447a39 4832
733d3e54 4833 if (MLX5_ESWITCH_MANAGER(priv->mdev))
1d447a39
SM		 4834 mlx5e_unregister_vport_reps(priv);
4835
6bfd390b 4836 mlx5e_disable_async_events(priv);
3deef8ce 4837 mlx5_lag_remove(mdev);
6bfd390b
HHZ		 4838}
4839
4840static const struct mlx5e_profile mlx5e_nic_profile = {
4841 .init = mlx5e_nic_init,
4842 .cleanup = mlx5e_nic_cleanup,
4843 .init_rx = mlx5e_init_nic_rx,
4844 .cleanup_rx = mlx5e_cleanup_nic_rx,
4845 .init_tx = mlx5e_init_nic_tx,
4846 .cleanup_tx = mlx5e_cleanup_nic_tx,
4847 .enable = mlx5e_nic_enable,
4848 .disable = mlx5e_nic_disable,
3834a5e6 4849 .update_stats = mlx5e_update_ndo_stats,
6bfd390b 4850 .max_nch = mlx5e_get_max_num_channels,
7ca42c80 4851 .update_carrier = mlx5e_update_carrier,
20fd0c19
SM		 4852 .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe,
4853 .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
6bfd390b
HHZ		 4854 .max_tc = MLX5E_MAX_NUM_TC,
4855};
4856
2c3b5bee
SM		 4857/* mlx5e generic netdev management API (move to en_common.c) */
4858
26e59d80
MHY		 4859struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
4860 const struct mlx5e_profile *profile,
4861 void *ppriv)
f62b8bb8 4862{
26e59d80 4863 int nch = profile->max_nch(mdev);
f62b8bb8
AV		 4864 struct net_device *netdev;
4865 struct mlx5e_priv *priv;
f62b8bb8 4866
08fb1dac 4867 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
6bfd390b 4868 nch * profile->max_tc,
08fb1dac 4869 nch);
f62b8bb8
AV		 4870 if (!netdev) {
4871 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
4872 return NULL;
4873 }
4874
be4891af
SM		 4875#ifdef CONFIG_RFS_ACCEL
4876 netdev->rx_cpu_rmap = mdev->rmap;
4877#endif
4878
127ea380 4879 profile->init(mdev, netdev, profile, ppriv);
f62b8bb8
AV		 4880
4881 netif_carrier_off(netdev);
4882
4883 priv = netdev_priv(netdev);
4884
7bb29755
MF		 4885 priv->wq = create_singlethread_workqueue("mlx5e");
4886 if (!priv->wq)
26e59d80
MHY		 4887 goto err_cleanup_nic;
4888
4889 return netdev;
4890
4891err_cleanup_nic:
31ac9338
OG		 4892 if (profile->cleanup)
4893 profile->cleanup(priv);
26e59d80
MHY		 4894 free_netdev(netdev);
4895
4896 return NULL;
4897}
4898
2c3b5bee 4899int mlx5e_attach_netdev(struct mlx5e_priv *priv)
26e59d80 4900{
2c3b5bee 4901 struct mlx5_core_dev *mdev = priv->mdev;
26e59d80 4902 const struct mlx5e_profile *profile;
26e59d80
MHY		 4903 int err;
4904
26e59d80
MHY		 4905 profile = priv->profile;
4906 clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
7bb29755 4907
6bfd390b
HHZ		 4908 err = profile->init_tx(priv);
4909 if (err)
ec8b9981 4910 goto out;
5c50368f 4911
7cbaf9a3
MS		 4912 mlx5e_create_q_counters(priv);
4913
4914 err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
5c50368f
AS		 4915 if (err) {
4916 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
7cbaf9a3 4917 goto err_destroy_q_counters;
5c50368f
AS		 4918 }
4919
6bfd390b
HHZ		 4920 err = profile->init_rx(priv);
4921 if (err)
5c50368f 4922 goto err_close_drop_rq;
5c50368f 4923
6bfd390b
HHZ		 4924 if (profile->enable)
4925 profile->enable(priv);
f62b8bb8 4926
26e59d80 4927 return 0;
5c50368f
AS		 4928
4929err_close_drop_rq:
a43b25da 4930 mlx5e_close_drop_rq(&priv->drop_rq);
5c50368f 4931
7cbaf9a3
MS		 4932err_destroy_q_counters:
4933 mlx5e_destroy_q_counters(priv);
6bfd390b 4934 profile->cleanup_tx(priv);
5c50368f 4935
26e59d80
MHY		 4936out:
4937 return err;
f62b8bb8
AV		 4938}
4939
2c3b5bee 4940void mlx5e_detach_netdev(struct mlx5e_priv *priv)
26e59d80 4941{
26e59d80
MHY		 4942 const struct mlx5e_profile *profile = priv->profile;
4943
4944 set_bit(MLX5E_STATE_DESTROYING, &priv->state);
26e59d80 4945
37f304d1
SM		 4946 if (profile->disable)
4947 profile->disable(priv);
4948 flush_workqueue(priv->wq);
4949
26e59d80 4950 profile->cleanup_rx(priv);
a43b25da 4951 mlx5e_close_drop_rq(&priv->drop_rq);
7cbaf9a3 4952 mlx5e_destroy_q_counters(priv);
26e59d80 4953 profile->cleanup_tx(priv);
26e59d80
MHY		 4954 cancel_delayed_work_sync(&priv->update_stats_work);
4955}
4956
2c3b5bee
SM		 4957void mlx5e_destroy_netdev(struct mlx5e_priv *priv)
4958{
4959 const struct mlx5e_profile *profile = priv->profile;
4960 struct net_device *netdev = priv->netdev;
4961
4962 destroy_workqueue(priv->wq);
4963 if (profile->cleanup)
4964 profile->cleanup(priv);
4965 free_netdev(netdev);
4966}
4967
26e59d80
MHY		 4968/* mlx5e_attach and mlx5e_detach scope should be only creating/destroying
4969 * hardware contexts and to connect it to the current netdev.
4970 */
4971static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv)
4972{
4973 struct mlx5e_priv *priv = vpriv;
4974 struct net_device *netdev = priv->netdev;
4975 int err;
4976
4977 if (netif_device_present(netdev))
4978 return 0;
4979
4980 err = mlx5e_create_mdev_resources(mdev);
4981 if (err)
4982 return err;
4983
2c3b5bee 4984 err = mlx5e_attach_netdev(priv);
26e59d80
MHY		 4985 if (err) {
4986 mlx5e_destroy_mdev_resources(mdev);
4987 return err;
4988 }
4989
4990 return 0;
4991}
4992
4993static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv)
4994{
4995 struct mlx5e_priv *priv = vpriv;
4996 struct net_device *netdev = priv->netdev;
4997
4998 if (!netif_device_present(netdev))
4999 return;
5000
2c3b5bee 5001 mlx5e_detach_netdev(priv);
26e59d80
MHY		 5002 mlx5e_destroy_mdev_resources(mdev);
5003}
5004
b50d292b
HHZ		 5005static void *mlx5e_add(struct mlx5_core_dev *mdev)
5006{
07c9f1e5
SM		 5007 struct net_device *netdev;
5008 void *rpriv = NULL;
26e59d80 5009 void *priv;
26e59d80 5010 int err;
b50d292b 5011
26e59d80
MHY		 5012 err = mlx5e_check_required_hca_cap(mdev);
5013 if (err)
b50d292b
HHZ		 5014 return NULL;
5015
e80541ec 5016#ifdef CONFIG_MLX5_ESWITCH
733d3e54 5017 if (MLX5_ESWITCH_MANAGER(mdev)) {
07c9f1e5 5018 rpriv = mlx5e_alloc_nic_rep_priv(mdev);
1d447a39 5019 if (!rpriv) {
07c9f1e5 5020 mlx5_core_warn(mdev, "Failed to alloc NIC rep priv data\n");
1d447a39
SM		 5021 return NULL;
5022 }
1d447a39 5023 }
e80541ec 5024#endif
127ea380 5025
1d447a39 5026 netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, rpriv);
26e59d80
MHY		 5027 if (!netdev) {
5028 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
07c9f1e5 5029 goto err_free_rpriv;
26e59d80
MHY		 5030 }
5031
5032 priv = netdev_priv(netdev);
5033
5034 err = mlx5e_attach(mdev, priv);
5035 if (err) {
5036 mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err);
5037 goto err_destroy_netdev;
5038 }
5039
5040 err = register_netdev(netdev);
5041 if (err) {
5042 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
5043 goto err_detach;
b50d292b 5044 }
26e59d80 5045
2a5e7a13
HN		 5046#ifdef CONFIG_MLX5_CORE_EN_DCB
5047 mlx5e_dcbnl_init_app(priv);
5048#endif
26e59d80
MHY		 5049 return priv;
5050
5051err_detach:
5052 mlx5e_detach(mdev, priv);
26e59d80 5053err_destroy_netdev:
2c3b5bee 5054 mlx5e_destroy_netdev(priv);
07c9f1e5 5055err_free_rpriv:
1d447a39 5056 kfree(rpriv);
26e59d80 5057 return NULL;
b50d292b
HHZ		 5058}
5059
b50d292b
HHZ		 5060static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv)
5061{
5062 struct mlx5e_priv *priv = vpriv;
1d447a39 5063 void *ppriv = priv->ppriv;
127ea380 5064
2a5e7a13
HN		 5065#ifdef CONFIG_MLX5_CORE_EN_DCB
5066 mlx5e_dcbnl_delete_app(priv);
5067#endif
5e1e93c7 5068 unregister_netdev(priv->netdev);
26e59d80 5069 mlx5e_detach(mdev, vpriv);
2c3b5bee 5070 mlx5e_destroy_netdev(priv);
1d447a39 5071 kfree(ppriv);
b50d292b
HHZ		 5072}
5073
f62b8bb8
AV		 5074static void *mlx5e_get_netdev(void *vpriv)
5075{
5076 struct mlx5e_priv *priv = vpriv;
5077
5078 return priv->netdev;
5079}
5080
5081static struct mlx5_interface mlx5e_interface = {
b50d292b
HHZ		 5082 .add = mlx5e_add,
5083 .remove = mlx5e_remove,
26e59d80
MHY		 5084 .attach = mlx5e_attach,
5085 .detach = mlx5e_detach,
f62b8bb8
AV		 5086 .event = mlx5e_async_event,
5087 .protocol = MLX5_INTERFACE_PROTOCOL_ETH,
5088 .get_dev = mlx5e_get_netdev,
5089};
5090
5091void mlx5e_init(void)
5092{
2ac9cfe7 5093 mlx5e_ipsec_build_inverse_table();
665bc539 5094 mlx5e_build_ptys2ethtool_map();
f62b8bb8
AV		 5095 mlx5_register_interface(&mlx5e_interface);
5096}
5097
5098void mlx5e_cleanup(void)
5099{
5100 mlx5_unregister_interface(&mlx5e_interface);
5101}
Mirror https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git