]> git.ipfire.org Git - people/arne_f/kernel.git/blame - drivers/net/ethernet/mellanox/mlx5/core/main.c
projects / people / arne_f / kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcp_memcontrol: Forward declare cgroup_subsys and mem_cgroup stucts
[people/arne_f/kernel.git] / drivers / net / ethernet / mellanox / mlx5 / core / main.c
CommitLineData
e126ba97 1/*
302bdf68 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
e126ba97
EC		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
adec640e 33#include <linux/highmem.h>
e126ba97
EC		 34#include <linux/module.h>
35#include <linux/init.h>
36#include <linux/errno.h>
37#include <linux/pci.h>
38#include <linux/dma-mapping.h>
39#include <linux/slab.h>
40#include <linux/io-mapping.h>
db058a18 41#include <linux/interrupt.h>
e3297246 42#include <linux/delay.h>
e126ba97
EC		 43#include <linux/mlx5/driver.h>
44#include <linux/mlx5/cq.h>
45#include <linux/mlx5/qp.h>
46#include <linux/mlx5/srq.h>
47#include <linux/debugfs.h>
f66f049f 48#include <linux/kmod.h>
89d44f0a 49#include <linux/delay.h>
b775516b 50#include <linux/mlx5/mlx5_ifc.h>
e126ba97 51#include "mlx5_core.h"
86d722ad 52#include "fs_core.h"
073bb189
SM		 53#ifdef CONFIG_MLX5_CORE_EN
54#include "eswitch.h"
55#endif
e126ba97 56
e126ba97 57MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
4ae6c18c 58MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver");
e126ba97
EC		 59MODULE_LICENSE("Dual BSD/GPL");
60MODULE_VERSION(DRIVER_VERSION);
61
62int mlx5_core_debug_mask;
63module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644);
64MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0");
65
9603b61d
JM		 66#define MLX5_DEFAULT_PROF 2
67static int prof_sel = MLX5_DEFAULT_PROF;
68module_param_named(prof_sel, prof_sel, int, 0444);
69MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
70
9603b61d
JM		 71static LIST_HEAD(intf_list);
72static LIST_HEAD(dev_list);
73static DEFINE_MUTEX(intf_mutex);
74
75struct mlx5_device_context {
76 struct list_head list;
77 struct mlx5_interface *intf;
78 void *context;
79};
80
81static struct mlx5_profile profile[] = {
82 [0] = {
83 .mask = 0,
84 },
85 [1] = {
86 .mask = MLX5_PROF_MASK_QP_SIZE,
87 .log_max_qp = 12,
88 },
89 [2] = {
90 .mask = MLX5_PROF_MASK_QP_SIZE |
91 MLX5_PROF_MASK_MR_CACHE,
92 .log_max_qp = 17,
93 .mr_cache[0] = {
94 .size = 500,
95 .limit = 250
96 },
97 .mr_cache[1] = {
98 .size = 500,
99 .limit = 250
100 },
101 .mr_cache[2] = {
102 .size = 500,
103 .limit = 250
104 },
105 .mr_cache[3] = {
106 .size = 500,
107 .limit = 250
108 },
109 .mr_cache[4] = {
110 .size = 500,
111 .limit = 250
112 },
113 .mr_cache[5] = {
114 .size = 500,
115 .limit = 250
116 },
117 .mr_cache[6] = {
118 .size = 500,
119 .limit = 250
120 },
121 .mr_cache[7] = {
122 .size = 500,
123 .limit = 250
124 },
125 .mr_cache[8] = {
126 .size = 500,
127 .limit = 250
128 },
129 .mr_cache[9] = {
130 .size = 500,
131 .limit = 250
132 },
133 .mr_cache[10] = {
134 .size = 500,
135 .limit = 250
136 },
137 .mr_cache[11] = {
138 .size = 500,
139 .limit = 250
140 },
141 .mr_cache[12] = {
142 .size = 64,
143 .limit = 32
144 },
145 .mr_cache[13] = {
146 .size = 32,
147 .limit = 16
148 },
149 .mr_cache[14] = {
150 .size = 16,
151 .limit = 8
152 },
153 .mr_cache[15] = {
154 .size = 8,
155 .limit = 4
156 },
157 },
158};
e126ba97 159
e3297246
EC		 160#define FW_INIT_TIMEOUT_MILI 2000
161#define FW_INIT_WAIT_MS 2
162
163static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili)
164{
165 unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili);
166 int err = 0;
167
168 while (fw_initializing(dev)) {
169 if (time_after(jiffies, end)) {
170 err = -EBUSY;
171 break;
172 }
173 msleep(FW_INIT_WAIT_MS);
174 }
175
176 return err;
177}
178
e126ba97
EC		 179static int set_dma_caps(struct pci_dev *pdev)
180{
181 int err;
182
183 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
184 if (err) {
1a91de28 185 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");
e126ba97
EC		 186 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
187 if (err) {
1a91de28 188 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
e126ba97
EC		 189 return err;
190 }
191 }
192
193 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
194 if (err) {
195 dev_warn(&pdev->dev,
1a91de28 196 "Warning: couldn't set 64-bit consistent PCI DMA mask\n");
e126ba97
EC		 197 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
198 if (err) {
199 dev_err(&pdev->dev,
1a91de28 200 "Can't set consistent PCI DMA mask, aborting\n");
e126ba97
EC		 201 return err;
202 }
203 }
204
205 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
206 return err;
207}
208
89d44f0a
MD		 209static int mlx5_pci_enable_device(struct mlx5_core_dev *dev)
210{
211 struct pci_dev *pdev = dev->pdev;
212 int err = 0;
213
214 mutex_lock(&dev->pci_status_mutex);
215 if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) {
216 err = pci_enable_device(pdev);
217 if (!err)
218 dev->pci_status = MLX5_PCI_STATUS_ENABLED;
219 }
220 mutex_unlock(&dev->pci_status_mutex);
221
222 return err;
223}
224
225static void mlx5_pci_disable_device(struct mlx5_core_dev *dev)
226{
227 struct pci_dev *pdev = dev->pdev;
228
229 mutex_lock(&dev->pci_status_mutex);
230 if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) {
231 pci_disable_device(pdev);
232 dev->pci_status = MLX5_PCI_STATUS_DISABLED;
233 }
234 mutex_unlock(&dev->pci_status_mutex);
235}
236
e126ba97
EC		 237static int request_bar(struct pci_dev *pdev)
238{
239 int err = 0;
240
241 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
1a91de28 242 dev_err(&pdev->dev, "Missing registers BAR, aborting\n");
e126ba97
EC		 243 return -ENODEV;
244 }
245
246 err = pci_request_regions(pdev, DRIVER_NAME);
247 if (err)
248 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");
249
250 return err;
251}
252
253static void release_bar(struct pci_dev *pdev)
254{
255 pci_release_regions(pdev);
256}
257
258static int mlx5_enable_msix(struct mlx5_core_dev *dev)
259{
db058a18
SM		 260 struct mlx5_priv *priv = &dev->priv;
261 struct mlx5_eq_table *table = &priv->eq_table;
938fe83c 262 int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq);
e126ba97 263 int nvec;
e126ba97
EC		 264 int i;
265
938fe83c
SM		 266 nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() +
267 MLX5_EQ_VEC_COMP_BASE;
e126ba97
EC		 268 nvec = min_t(int, nvec, num_eqs);
269 if (nvec <= MLX5_EQ_VEC_COMP_BASE)
270 return -ENOMEM;
271
db058a18
SM		 272 priv->msix_arr = kcalloc(nvec, sizeof(*priv->msix_arr), GFP_KERNEL);
273
274 priv->irq_info = kcalloc(nvec, sizeof(*priv->irq_info), GFP_KERNEL);
275 if (!priv->msix_arr || !priv->irq_info)
276 goto err_free_msix;
e126ba97
EC		 277
278 for (i = 0; i < nvec; i++)
db058a18 279 priv->msix_arr[i].entry = i;
e126ba97 280
db058a18 281 nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr,
3a9e161a 282 MLX5_EQ_VEC_COMP_BASE + 1, nvec);
f3c9407b
AG		 283 if (nvec < 0)
284 return nvec;
e126ba97 285
f3c9407b 286 table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE;
e126ba97
EC		 287
288 return 0;
db058a18
SM		 289
290err_free_msix:
291 kfree(priv->irq_info);
292 kfree(priv->msix_arr);
293 return -ENOMEM;
e126ba97
EC		 294}
295
296static void mlx5_disable_msix(struct mlx5_core_dev *dev)
297{
db058a18 298 struct mlx5_priv *priv = &dev->priv;
e126ba97
EC		 299
300 pci_disable_msix(dev->pdev);
db058a18
SM		 301 kfree(priv->irq_info);
302 kfree(priv->msix_arr);
e126ba97
EC		 303}
304
305struct mlx5_reg_host_endianess {
306 u8 he;
307 u8 rsvd[15];
308};
309
87b8de49
EC		 310
311#define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos))
312
313enum {
c7a08ac7
EC		 314 MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) |
315 MLX5_DEV_CAP_FLAG_DCT,
87b8de49
EC		 316};
317
c7a08ac7
EC		 318static u16 to_fw_pkey_sz(u32 size)
319{
320 switch (size) {
321 case 128:
322 return 0;
323 case 256:
324 return 1;
325 case 512:
326 return 2;
327 case 1024:
328 return 3;
329 case 2048:
330 return 4;
331 case 4096:
332 return 5;
333 default:
334 pr_warn("invalid pkey table size %d\n", size);
335 return 0;
336 }
337}
338
938fe83c
SM		 339int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type,
340 enum mlx5_cap_mode cap_mode)
c7a08ac7 341{
b775516b
EC		 342 u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)];
343 int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
938fe83c
SM		 344 void *out, *hca_caps;
345 u16 opmod = (cap_type << 1) | (cap_mode & 0x01);
e126ba97
EC		 346 int err;
347
b775516b
EC		 348 memset(in, 0, sizeof(in));
349 out = kzalloc(out_sz, GFP_KERNEL);
c7a08ac7 350 if (!out)
e126ba97 351 return -ENOMEM;
938fe83c 352
b775516b
EC		 353 MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
354 MLX5_SET(query_hca_cap_in, in, op_mod, opmod);
355 err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz);
356 if (err)
357 goto query_ex;
e126ba97 358
b775516b 359 err = mlx5_cmd_status_to_err_v2(out);
c7a08ac7 360 if (err) {
938fe83c
SM		 361 mlx5_core_warn(dev,
362 "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n",
363 cap_type, cap_mode, err);
e126ba97
EC		 364 goto query_ex;
365 }
c7a08ac7 366
938fe83c
SM		 367 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability);
368
369 switch (cap_mode) {
370 case HCA_CAP_OPMOD_GET_MAX:
371 memcpy(dev->hca_caps_max[cap_type], hca_caps,
372 MLX5_UN_SZ_BYTES(hca_cap_union));
373 break;
374 case HCA_CAP_OPMOD_GET_CUR:
375 memcpy(dev->hca_caps_cur[cap_type], hca_caps,
376 MLX5_UN_SZ_BYTES(hca_cap_union));
377 break;
378 default:
379 mlx5_core_warn(dev,
380 "Tried to query dev cap type(%x) with wrong opmode(%x)\n",
381 cap_type, cap_mode);
382 err = -EINVAL;
383 break;
384 }
c7a08ac7
EC		 385query_ex:
386 kfree(out);
387 return err;
388}
389
b775516b 390static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz)
c7a08ac7 391{
b775516b 392 u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)];
c7a08ac7
EC		 393 int err;
394
b775516b 395 memset(out, 0, sizeof(out));
e126ba97 396
b775516b
EC		 397 MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP);
398 err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
e126ba97 399 if (err)
c7a08ac7 400 return err;
e126ba97 401
b775516b 402 err = mlx5_cmd_status_to_err_v2(out);
c7a08ac7
EC		 403
404 return err;
405}
406
407static int handle_hca_cap(struct mlx5_core_dev *dev)
408{
b775516b 409 void *set_ctx = NULL;
c7a08ac7 410 struct mlx5_profile *prof = dev->profile;
c7a08ac7 411 int err = -ENOMEM;
b775516b 412 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
938fe83c 413 void *set_hca_cap;
c7a08ac7 414
b775516b 415 set_ctx = kzalloc(set_sz, GFP_KERNEL);
c7a08ac7 416 if (!set_ctx)
e126ba97 417 goto query_ex;
e126ba97 418
938fe83c 419 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_MAX);
c7a08ac7 420 if (err)
e126ba97 421 goto query_ex;
e126ba97 422
938fe83c 423 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_CUR);
e126ba97
EC		 424 if (err)
425 goto query_ex;
426
938fe83c
SM		 427 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx,
428 capability);
429 memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL],
430 MLX5_ST_SZ_BYTES(cmd_hca_cap));
431
432 mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n",
707c4602 433 mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)),
938fe83c 434 128);
c7a08ac7 435 /* we limit the size of the pkey table to 128 entries for now */
938fe83c
SM		 436 MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size,
437 to_fw_pkey_sz(128));
c7a08ac7
EC		 438
439 if (prof->mask & MLX5_PROF_MASK_QP_SIZE)
938fe83c
SM		 440 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp,
441 prof->log_max_qp);
c7a08ac7 442
938fe83c
SM		 443 /* disable cmdif checksum */
444 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0);
c7a08ac7 445
fe1e1876
CS		 446 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12);
447
b775516b 448 err = set_caps(dev, set_ctx, set_sz);
c7a08ac7 449
e126ba97 450query_ex:
e126ba97 451 kfree(set_ctx);
e126ba97
EC		 452 return err;
453}
454
455static int set_hca_ctrl(struct mlx5_core_dev *dev)
456{
457 struct mlx5_reg_host_endianess he_in;
458 struct mlx5_reg_host_endianess he_out;
459 int err;
460
fc50db98
EC		 461 if (!mlx5_core_is_pf(dev))
462 return 0;
463
e126ba97
EC		 464 memset(&he_in, 0, sizeof(he_in));
465 he_in.he = MLX5_SET_HOST_ENDIANNESS;
466 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in),
467 &he_out, sizeof(he_out),
468 MLX5_REG_HOST_ENDIANNESS, 0, 1);
469 return err;
470}
471
0b107106 472int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id)
cd23b14b 473{
0b107106
EC		 474 u32 out[MLX5_ST_SZ_DW(enable_hca_out)];
475 u32 in[MLX5_ST_SZ_DW(enable_hca_in)];
cd23b14b 476 int err;
cd23b14b 477
0b107106
EC		 478 memset(in, 0, sizeof(in));
479 MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA);
480 MLX5_SET(enable_hca_in, in, function_id, func_id);
481 memset(out, 0, sizeof(out));
482
cd23b14b
EC		 483 err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
484 if (err)
485 return err;
486
0b107106 487 return mlx5_cmd_status_to_err_v2(out);
cd23b14b
EC		 488}
489
0b107106 490int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id)
cd23b14b 491{
0b107106
EC		 492 u32 out[MLX5_ST_SZ_DW(disable_hca_out)];
493 u32 in[MLX5_ST_SZ_DW(disable_hca_in)];
cd23b14b 494 int err;
cd23b14b 495
0b107106
EC		 496 memset(in, 0, sizeof(in));
497 MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
498 MLX5_SET(disable_hca_in, in, function_id, func_id);
499 memset(out, 0, sizeof(out));
500 err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
cd23b14b
EC		 501 if (err)
502 return err;
503
0b107106 504 return mlx5_cmd_status_to_err_v2(out);
cd23b14b
EC		 505}
506
b0844444
EBE		 507cycle_t mlx5_read_internal_timer(struct mlx5_core_dev *dev)
508{
509 u32 timer_h, timer_h1, timer_l;
510
511 timer_h = ioread32be(&dev->iseg->internal_timer_h);
512 timer_l = ioread32be(&dev->iseg->internal_timer_l);
513 timer_h1 = ioread32be(&dev->iseg->internal_timer_h);
514 if (timer_h != timer_h1) /* wrap around */
515 timer_l = ioread32be(&dev->iseg->internal_timer_l);
516
517 return (cycle_t)timer_l | (cycle_t)timer_h1 << 32;
518}
519
db058a18
SM		 520static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i)
521{
522 struct mlx5_priv *priv = &mdev->priv;
523 struct msix_entry *msix = priv->msix_arr;
524 int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector;
311c7c71 525 int numa_node = priv->numa_node;
db058a18
SM		 526 int err;
527
528 if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) {
529 mlx5_core_warn(mdev, "zalloc_cpumask_var failed");
530 return -ENOMEM;
531 }
532
dda922c8
DM		 533 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
534 priv->irq_info[i].mask);
db058a18
SM		 535
536 err = irq_set_affinity_hint(irq, priv->irq_info[i].mask);
537 if (err) {
538 mlx5_core_warn(mdev, "irq_set_affinity_hint failed,irq 0x%.4x",
539 irq);
540 goto err_clear_mask;
541 }
542
543 return 0;
544
545err_clear_mask:
546 free_cpumask_var(priv->irq_info[i].mask);
547 return err;
548}
549
550static void mlx5_irq_clear_affinity_hint(struct mlx5_core_dev *mdev, int i)
551{
552 struct mlx5_priv *priv = &mdev->priv;
553 struct msix_entry *msix = priv->msix_arr;
554 int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector;
555
556 irq_set_affinity_hint(irq, NULL);
557 free_cpumask_var(priv->irq_info[i].mask);
558}
559
560static int mlx5_irq_set_affinity_hints(struct mlx5_core_dev *mdev)
561{
562 int err;
563 int i;
564
565 for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) {
566 err = mlx5_irq_set_affinity_hint(mdev, i);
567 if (err)
568 goto err_out;
569 }
570
571 return 0;
572
573err_out:
574 for (i--; i >= 0; i--)
575 mlx5_irq_clear_affinity_hint(mdev, i);
576
577 return err;
578}
579
580static void mlx5_irq_clear_affinity_hints(struct mlx5_core_dev *mdev)
581{
582 int i;
583
584 for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++)
585 mlx5_irq_clear_affinity_hint(mdev, i);
586}
587
233d05d2
SM		 588int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn)
589{
590 struct mlx5_eq_table *table = &dev->priv.eq_table;
591 struct mlx5_eq *eq, *n;
592 int err = -ENOENT;
593
594 spin_lock(&table->lock);
595 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
596 if (eq->index == vector) {
597 *eqn = eq->eqn;
598 *irqn = eq->irqn;
599 err = 0;
600 break;
601 }
602 }
603 spin_unlock(&table->lock);
604
605 return err;
606}
607EXPORT_SYMBOL(mlx5_vector2eqn);
608
609static void free_comp_eqs(struct mlx5_core_dev *dev)
610{
611 struct mlx5_eq_table *table = &dev->priv.eq_table;
612 struct mlx5_eq *eq, *n;
613
614 spin_lock(&table->lock);
615 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
616 list_del(&eq->list);
617 spin_unlock(&table->lock);
618 if (mlx5_destroy_unmap_eq(dev, eq))
619 mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n",
620 eq->eqn);
621 kfree(eq);
622 spin_lock(&table->lock);
623 }
624 spin_unlock(&table->lock);
625}
626
627static int alloc_comp_eqs(struct mlx5_core_dev *dev)
628{
629 struct mlx5_eq_table *table = &dev->priv.eq_table;
db058a18 630 char name[MLX5_MAX_IRQ_NAME];
233d05d2
SM		 631 struct mlx5_eq *eq;
632 int ncomp_vec;
633 int nent;
634 int err;
635 int i;
636
637 INIT_LIST_HEAD(&table->comp_eqs_list);
638 ncomp_vec = table->num_comp_vectors;
639 nent = MLX5_COMP_EQ_SIZE;
640 for (i = 0; i < ncomp_vec; i++) {
641 eq = kzalloc(sizeof(*eq), GFP_KERNEL);
642 if (!eq) {
643 err = -ENOMEM;
644 goto clean;
645 }
646
db058a18 647 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i);
233d05d2
SM		 648 err = mlx5_create_map_eq(dev, eq,
649 i + MLX5_EQ_VEC_COMP_BASE, nent, 0,
650 name, &dev->priv.uuari.uars[0]);
651 if (err) {
652 kfree(eq);
653 goto clean;
654 }
655 mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn);
656 eq->index = i;
657 spin_lock(&table->lock);
658 list_add_tail(&eq->list, &table->comp_eqs_list);
659 spin_unlock(&table->lock);
660 }
661
662 return 0;
663
664clean:
665 free_comp_eqs(dev);
666 return err;
667}
668
f62b8bb8
AV		 669#ifdef CONFIG_MLX5_CORE_EN
670static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
671{
672 u32 query_in[MLX5_ST_SZ_DW(query_issi_in)];
673 u32 query_out[MLX5_ST_SZ_DW(query_issi_out)];
674 u32 set_in[MLX5_ST_SZ_DW(set_issi_in)];
675 u32 set_out[MLX5_ST_SZ_DW(set_issi_out)];
676 int err;
677 u32 sup_issi;
678
679 memset(query_in, 0, sizeof(query_in));
680 memset(query_out, 0, sizeof(query_out));
681
682 MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI);
683
684 err = mlx5_cmd_exec_check_status(dev, query_in, sizeof(query_in),
685 query_out, sizeof(query_out));
686 if (err) {
687 if (((struct mlx5_outbox_hdr *)query_out)->status ==
688 MLX5_CMD_STAT_BAD_OP_ERR) {
689 pr_debug("Only ISSI 0 is supported\n");
690 return 0;
691 }
692
693 pr_err("failed to query ISSI\n");
694 return err;
695 }
696
697 sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0);
698
699 if (sup_issi & (1 << 1)) {
700 memset(set_in, 0, sizeof(set_in));
701 memset(set_out, 0, sizeof(set_out));
702
703 MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI);
704 MLX5_SET(set_issi_in, set_in, current_issi, 1);
705
706 err = mlx5_cmd_exec_check_status(dev, set_in, sizeof(set_in),
707 set_out, sizeof(set_out));
708 if (err) {
709 pr_err("failed to set ISSI=1\n");
710 return err;
711 }
712
713 dev->issi = 1;
714
715 return 0;
e74a1db0 716 } else if (sup_issi & (1 << 0) || !sup_issi) {
f62b8bb8
AV		 717 return 0;
718 }
719
720 return -ENOTSUPP;
721}
722#endif
723
88a85f99
AS		 724static int map_bf_area(struct mlx5_core_dev *dev)
725{
726 resource_size_t bf_start = pci_resource_start(dev->pdev, 0);
727 resource_size_t bf_len = pci_resource_len(dev->pdev, 0);
728
729 dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len);
730
731 return dev->priv.bf_mapping ? 0 : -ENOMEM;
732}
733
734static void unmap_bf_area(struct mlx5_core_dev *dev)
735{
736 if (dev->priv.bf_mapping)
737 io_mapping_free(dev->priv.bf_mapping);
738}
739
a31208b1
MD		 740static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
741{
742 struct mlx5_device_context *dev_ctx;
743 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
744
745 dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL);
746 if (!dev_ctx)
747 return;
748
749 dev_ctx->intf = intf;
750 dev_ctx->context = intf->add(dev);
751
752 if (dev_ctx->context) {
753 spin_lock_irq(&priv->ctx_lock);
754 list_add_tail(&dev_ctx->list, &priv->ctx_list);
755 spin_unlock_irq(&priv->ctx_lock);
756 } else {
757 kfree(dev_ctx);
758 }
759}
760
761static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
762{
763 struct mlx5_device_context *dev_ctx;
764 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
765
766 list_for_each_entry(dev_ctx, &priv->ctx_list, list)
767 if (dev_ctx->intf == intf) {
768 spin_lock_irq(&priv->ctx_lock);
769 list_del(&dev_ctx->list);
770 spin_unlock_irq(&priv->ctx_lock);
771
772 intf->remove(dev, dev_ctx->context);
773 kfree(dev_ctx);
774 return;
775 }
776}
777
778static int mlx5_register_device(struct mlx5_core_dev *dev)
e126ba97
EC		 779{
780 struct mlx5_priv *priv = &dev->priv;
a31208b1
MD		 781 struct mlx5_interface *intf;
782
783 mutex_lock(&intf_mutex);
784 list_add_tail(&priv->dev_list, &dev_list);
785 list_for_each_entry(intf, &intf_list, list)
786 mlx5_add_device(intf, priv);
787 mutex_unlock(&intf_mutex);
788
789 return 0;
790}
791
792static void mlx5_unregister_device(struct mlx5_core_dev *dev)
793{
794 struct mlx5_priv *priv = &dev->priv;
795 struct mlx5_interface *intf;
796
797 mutex_lock(&intf_mutex);
798 list_for_each_entry(intf, &intf_list, list)
799 mlx5_remove_device(intf, priv);
800 list_del(&priv->dev_list);
801 mutex_unlock(&intf_mutex);
802}
803
804int mlx5_register_interface(struct mlx5_interface *intf)
805{
806 struct mlx5_priv *priv;
807
808 if (!intf->add || !intf->remove)
809 return -EINVAL;
810
811 mutex_lock(&intf_mutex);
812 list_add_tail(&intf->list, &intf_list);
813 list_for_each_entry(priv, &dev_list, dev_list)
814 mlx5_add_device(intf, priv);
815 mutex_unlock(&intf_mutex);
816
817 return 0;
818}
819EXPORT_SYMBOL(mlx5_register_interface);
820
821void mlx5_unregister_interface(struct mlx5_interface *intf)
822{
823 struct mlx5_priv *priv;
824
825 mutex_lock(&intf_mutex);
826 list_for_each_entry(priv, &dev_list, dev_list)
827 mlx5_remove_device(intf, priv);
828 list_del(&intf->list);
829 mutex_unlock(&intf_mutex);
830}
831EXPORT_SYMBOL(mlx5_unregister_interface);
832
833void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol)
834{
835 struct mlx5_priv *priv = &mdev->priv;
836 struct mlx5_device_context *dev_ctx;
837 unsigned long flags;
838 void *result = NULL;
839
840 spin_lock_irqsave(&priv->ctx_lock, flags);
841
842 list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list)
843 if ((dev_ctx->intf->protocol == protocol) &&
844 dev_ctx->intf->get_dev) {
845 result = dev_ctx->intf->get_dev(dev_ctx->context);
846 break;
847 }
848
849 spin_unlock_irqrestore(&priv->ctx_lock, flags);
850
851 return result;
852}
853EXPORT_SYMBOL(mlx5_get_protocol_dev);
854
855static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
856{
857 struct pci_dev *pdev = dev->pdev;
858 int err = 0;
e126ba97 859
e126ba97
EC		 860 pci_set_drvdata(dev->pdev, dev);
861 strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN);
862 priv->name[MLX5_MAX_NAME_LEN - 1] = 0;
863
864 mutex_init(&priv->pgdir_mutex);
865 INIT_LIST_HEAD(&priv->pgdir_list);
866 spin_lock_init(&priv->mkey_lock);
867
311c7c71
SM		 868 mutex_init(&priv->alloc_mutex);
869
870 priv->numa_node = dev_to_node(&dev->pdev->dev);
871
e126ba97
EC		 872 priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root);
873 if (!priv->dbg_root)
874 return -ENOMEM;
875
89d44f0a 876 err = mlx5_pci_enable_device(dev);
e126ba97 877 if (err) {
1a91de28 878 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
e126ba97
EC		 879 goto err_dbg;
880 }
881
882 err = request_bar(pdev);
883 if (err) {
1a91de28 884 dev_err(&pdev->dev, "error requesting BARs, aborting\n");
e126ba97
EC		 885 goto err_disable;
886 }
887
888 pci_set_master(pdev);
889
890 err = set_dma_caps(pdev);
891 if (err) {
892 dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n");
893 goto err_clr_master;
894 }
895
896 dev->iseg_base = pci_resource_start(dev->pdev, 0);
897 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg));
898 if (!dev->iseg) {
899 err = -ENOMEM;
900 dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n");
901 goto err_clr_master;
902 }
a31208b1
MD		 903
904 return 0;
905
906err_clr_master:
907 pci_clear_master(dev->pdev);
908 release_bar(dev->pdev);
909err_disable:
89d44f0a 910 mlx5_pci_disable_device(dev);
a31208b1
MD		 911
912err_dbg:
913 debugfs_remove(priv->dbg_root);
914 return err;
915}
916
917static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
918{
919 iounmap(dev->iseg);
920 pci_clear_master(dev->pdev);
921 release_bar(dev->pdev);
89d44f0a 922 mlx5_pci_disable_device(dev);
a31208b1
MD		 923 debugfs_remove(priv->dbg_root);
924}
925
926#define MLX5_IB_MOD "mlx5_ib"
927static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
928{
929 struct pci_dev *pdev = dev->pdev;
930 int err;
931
89d44f0a
MD		 932 mutex_lock(&dev->intf_state_mutex);
933 if (dev->interface_state == MLX5_INTERFACE_STATE_UP) {
934 dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n",
935 __func__);
936 goto out;
937 }
938
e126ba97
EC		 939 dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev),
940 fw_rev_min(dev), fw_rev_sub(dev));
941
89d44f0a
MD		 942 /* on load removing any previous indication of internal error, device is
943 * up
944 */
945 dev->state = MLX5_DEVICE_STATE_UP;
946
e126ba97
EC		 947 err = mlx5_cmd_init(dev);
948 if (err) {
949 dev_err(&pdev->dev, "Failed initializing command interface, aborting\n");
89d44f0a 950 goto out_err;
e126ba97
EC		 951 }
952
e3297246
EC		 953 err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI);
954 if (err) {
955 dev_err(&dev->pdev->dev, "Firmware over %d MS in initializing state, aborting\n",
956 FW_INIT_TIMEOUT_MILI);
957 goto out_err;
958 }
959
e126ba97 960 mlx5_pagealloc_init(dev);
cd23b14b 961
0b107106 962 err = mlx5_core_enable_hca(dev, 0);
cd23b14b
EC		 963 if (err) {
964 dev_err(&pdev->dev, "enable hca failed\n");
965 goto err_pagealloc_cleanup;
966 }
967
f62b8bb8
AV		 968#ifdef CONFIG_MLX5_CORE_EN
969 err = mlx5_core_set_issi(dev);
970 if (err) {
971 dev_err(&pdev->dev, "failed to set issi\n");
972 goto err_disable_hca;
973 }
974#endif
975
cd23b14b
EC		 976 err = mlx5_satisfy_startup_pages(dev, 1);
977 if (err) {
978 dev_err(&pdev->dev, "failed to allocate boot pages\n");
979 goto err_disable_hca;
980 }
981
e126ba97
EC		 982 err = set_hca_ctrl(dev);
983 if (err) {
984 dev_err(&pdev->dev, "set_hca_ctrl failed\n");
cd23b14b 985 goto reclaim_boot_pages;
e126ba97
EC		 986 }
987
988 err = handle_hca_cap(dev);
989 if (err) {
990 dev_err(&pdev->dev, "handle_hca_cap failed\n");
cd23b14b 991 goto reclaim_boot_pages;
e126ba97
EC		 992 }
993
cd23b14b 994 err = mlx5_satisfy_startup_pages(dev, 0);
e126ba97 995 if (err) {
cd23b14b
EC		 996 dev_err(&pdev->dev, "failed to allocate init pages\n");
997 goto reclaim_boot_pages;
e126ba97
EC		 998 }
999
1000 err = mlx5_pagealloc_start(dev);
1001 if (err) {
1002 dev_err(&pdev->dev, "mlx5_pagealloc_start failed\n");
cd23b14b 1003 goto reclaim_boot_pages;
e126ba97
EC		 1004 }
1005
1006 err = mlx5_cmd_init_hca(dev);
1007 if (err) {
1008 dev_err(&pdev->dev, "init hca failed\n");
1009 goto err_pagealloc_stop;
1010 }
1011
1012 mlx5_start_health_poll(dev);
1013
938fe83c 1014 err = mlx5_query_hca_caps(dev);
e126ba97
EC		 1015 if (err) {
1016 dev_err(&pdev->dev, "query hca failed\n");
1017 goto err_stop_poll;
1018 }
1019
211e6c80 1020 err = mlx5_query_board_id(dev);
e126ba97 1021 if (err) {
211e6c80 1022 dev_err(&pdev->dev, "query board id failed\n");
e126ba97
EC		 1023 goto err_stop_poll;
1024 }
1025
1026 err = mlx5_enable_msix(dev);
1027 if (err) {
1028 dev_err(&pdev->dev, "enable msix failed\n");
1029 goto err_stop_poll;
1030 }
1031
1032 err = mlx5_eq_init(dev);
1033 if (err) {
1034 dev_err(&pdev->dev, "failed to initialize eq\n");
1035 goto disable_msix;
1036 }
1037
1038 err = mlx5_alloc_uuars(dev, &priv->uuari);
1039 if (err) {
1040 dev_err(&pdev->dev, "Failed allocating uar, aborting\n");
1041 goto err_eq_cleanup;
1042 }
1043
1044 err = mlx5_start_eqs(dev);
1045 if (err) {
1046 dev_err(&pdev->dev, "Failed to start pages and async EQs\n");
1047 goto err_free_uar;
1048 }
1049
233d05d2
SM		 1050 err = alloc_comp_eqs(dev);
1051 if (err) {
1052 dev_err(&pdev->dev, "Failed to alloc completion EQs\n");
1053 goto err_stop_eqs;
1054 }
1055
88a85f99
AS		 1056 if (map_bf_area(dev))
1057 dev_err(&pdev->dev, "Failed to map blue flame area\n");
1058
db058a18
SM		 1059 err = mlx5_irq_set_affinity_hints(dev);
1060 if (err) {
1061 dev_err(&pdev->dev, "Failed to alloc affinity hint cpumask\n");
88a85f99 1062 goto err_unmap_bf_area;
db058a18
SM		 1063 }
1064
e126ba97
EC		 1065 MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock);
1066
1067 mlx5_init_cq_table(dev);
1068 mlx5_init_qp_table(dev);
1069 mlx5_init_srq_table(dev);
3bcdb17a 1070 mlx5_init_mr_table(dev);
e126ba97 1071
86d722ad
MG		 1072 err = mlx5_init_fs(dev);
1073 if (err) {
1074 dev_err(&pdev->dev, "Failed to init flow steering\n");
1075 goto err_fs;
1076 }
073bb189
SM		 1077#ifdef CONFIG_MLX5_CORE_EN
1078 err = mlx5_eswitch_init(dev);
1079 if (err) {
1080 dev_err(&pdev->dev, "eswitch init failed %d\n", err);
1081 goto err_reg_dev;
1082 }
1083#endif
1084
fc50db98
EC		 1085 err = mlx5_sriov_init(dev);
1086 if (err) {
1087 dev_err(&pdev->dev, "sriov init failed %d\n", err);
1088 goto err_sriov;
1089 }
1090
a31208b1
MD		 1091 err = mlx5_register_device(dev);
1092 if (err) {
1093 dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err);
1094 goto err_reg_dev;
1095 }
1096
1097 err = request_module_nowait(MLX5_IB_MOD);
1098 if (err)
1099 pr_info("failed request module on %s\n", MLX5_IB_MOD);
1100
89d44f0a
MD		 1101 dev->interface_state = MLX5_INTERFACE_STATE_UP;
1102out:
1103 mutex_unlock(&dev->intf_state_mutex);
1104
e126ba97
EC		 1105 return 0;
1106
fc50db98
EC		 1107err_sriov:
1108 if (mlx5_sriov_cleanup(dev))
1109 dev_err(&dev->pdev->dev, "sriov cleanup failed\n");
1110
073bb189
SM		 1111#ifdef CONFIG_MLX5_CORE_EN
1112 mlx5_eswitch_cleanup(dev->priv.eswitch);
1113#endif
a31208b1 1114err_reg_dev:
86d722ad
MG		 1115 mlx5_cleanup_fs(dev);
1116err_fs:
a31208b1
MD		 1117 mlx5_cleanup_mr_table(dev);
1118 mlx5_cleanup_srq_table(dev);
1119 mlx5_cleanup_qp_table(dev);
1120 mlx5_cleanup_cq_table(dev);
1121 mlx5_irq_clear_affinity_hints(dev);
1122
88a85f99
AS		 1123err_unmap_bf_area:
1124 unmap_bf_area(dev);
1125
db058a18
SM		 1126 free_comp_eqs(dev);
1127
233d05d2
SM		 1128err_stop_eqs:
1129 mlx5_stop_eqs(dev);
1130
e126ba97
EC		 1131err_free_uar:
1132 mlx5_free_uuars(dev, &priv->uuari);
1133
1134err_eq_cleanup:
1135 mlx5_eq_cleanup(dev);
1136
1137disable_msix:
1138 mlx5_disable_msix(dev);
1139
1140err_stop_poll:
1141 mlx5_stop_health_poll(dev);
1bde6e30
EC		 1142 if (mlx5_cmd_teardown_hca(dev)) {
1143 dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n");
89d44f0a 1144 goto out_err;
1bde6e30 1145 }
e126ba97
EC		 1146
1147err_pagealloc_stop:
1148 mlx5_pagealloc_stop(dev);
1149
cd23b14b 1150reclaim_boot_pages:
e126ba97
EC		 1151 mlx5_reclaim_startup_pages(dev);
1152
cd23b14b 1153err_disable_hca:
0b107106 1154 mlx5_core_disable_hca(dev, 0);
cd23b14b 1155
e126ba97
EC		 1156err_pagealloc_cleanup:
1157 mlx5_pagealloc_cleanup(dev);
1158 mlx5_cmd_cleanup(dev);
1159
89d44f0a
MD		 1160out_err:
1161 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1162 mutex_unlock(&dev->intf_state_mutex);
1163
e126ba97
EC		 1164 return err;
1165}
e126ba97 1166
a31208b1 1167static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
e126ba97 1168{
89d44f0a 1169 int err = 0;
e126ba97 1170
fc50db98
EC		 1171 err = mlx5_sriov_cleanup(dev);
1172 if (err) {
1173 dev_warn(&dev->pdev->dev, "%s: sriov cleanup failed - abort\n",
1174 __func__);
1175 return err;
1176 }
1177
89d44f0a
MD		 1178 mutex_lock(&dev->intf_state_mutex);
1179 if (dev->interface_state == MLX5_INTERFACE_STATE_DOWN) {
1180 dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n",
1181 __func__);
1182 goto out;
1183 }
a31208b1 1184 mlx5_unregister_device(dev);
073bb189
SM		 1185#ifdef CONFIG_MLX5_CORE_EN
1186 mlx5_eswitch_cleanup(dev->priv.eswitch);
1187#endif
1188
86d722ad 1189 mlx5_cleanup_fs(dev);
a31208b1 1190 mlx5_cleanup_mr_table(dev);
e126ba97
EC		 1191 mlx5_cleanup_srq_table(dev);
1192 mlx5_cleanup_qp_table(dev);
1193 mlx5_cleanup_cq_table(dev);
db058a18 1194 mlx5_irq_clear_affinity_hints(dev);
88a85f99 1195 unmap_bf_area(dev);
233d05d2 1196 free_comp_eqs(dev);
e126ba97
EC		 1197 mlx5_stop_eqs(dev);
1198 mlx5_free_uuars(dev, &priv->uuari);
1199 mlx5_eq_cleanup(dev);
1200 mlx5_disable_msix(dev);
1201 mlx5_stop_health_poll(dev);
ac6ea6e8
EC		 1202 err = mlx5_cmd_teardown_hca(dev);
1203 if (err) {
1bde6e30 1204 dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n");
ac6ea6e8 1205 goto out;
1bde6e30 1206 }
e126ba97
EC		 1207 mlx5_pagealloc_stop(dev);
1208 mlx5_reclaim_startup_pages(dev);
0b107106 1209 mlx5_core_disable_hca(dev, 0);
e126ba97
EC		 1210 mlx5_pagealloc_cleanup(dev);
1211 mlx5_cmd_cleanup(dev);
9603b61d 1212
ac6ea6e8 1213out:
89d44f0a
MD		 1214 dev->interface_state = MLX5_INTERFACE_STATE_DOWN;
1215 mutex_unlock(&dev->intf_state_mutex);
ac6ea6e8 1216 return err;
9603b61d 1217}
64613d94 1218
89d44f0a 1219void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
ac6ea6e8 1220 unsigned long param)
9603b61d
JM		 1221{
1222 struct mlx5_priv *priv = &dev->priv;
1223 struct mlx5_device_context *dev_ctx;
1224 unsigned long flags;
1225
1226 spin_lock_irqsave(&priv->ctx_lock, flags);
1227
1228 list_for_each_entry(dev_ctx, &priv->ctx_list, list)
1229 if (dev_ctx->intf->event)
4d2f9bbb 1230 dev_ctx->intf->event(dev, dev_ctx->context, event, param);
9603b61d
JM		 1231
1232 spin_unlock_irqrestore(&priv->ctx_lock, flags);
1233}
1234
1235struct mlx5_core_event_handler {
1236 void (*event)(struct mlx5_core_dev *dev,
1237 enum mlx5_dev_event event,
1238 void *data);
1239};
1240
f66f049f 1241
9603b61d
JM		 1242static int init_one(struct pci_dev *pdev,
1243 const struct pci_device_id *id)
1244{
1245 struct mlx5_core_dev *dev;
1246 struct mlx5_priv *priv;
1247 int err;
1248
1249 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1250 if (!dev) {
1251 dev_err(&pdev->dev, "kzalloc failed\n");
1252 return -ENOMEM;
1253 }
1254 priv = &dev->priv;
fc50db98 1255 priv->pci_dev_data = id->driver_data;
9603b61d
JM		 1256
1257 pci_set_drvdata(pdev, dev);
1258
1259 if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profile)) {
1260 pr_warn("selected profile out of range, selecting default (%d)\n",
1261 MLX5_DEFAULT_PROF);
1262 prof_sel = MLX5_DEFAULT_PROF;
1263 }
1264 dev->profile = &profile[prof_sel];
a31208b1 1265 dev->pdev = pdev;
9603b61d
JM		 1266 dev->event = mlx5_core_event;
1267
364d1798
EC		 1268 INIT_LIST_HEAD(&priv->ctx_list);
1269 spin_lock_init(&priv->ctx_lock);
89d44f0a
MD		 1270 mutex_init(&dev->pci_status_mutex);
1271 mutex_init(&dev->intf_state_mutex);
a31208b1 1272 err = mlx5_pci_init(dev, priv);
9603b61d 1273 if (err) {
a31208b1
MD		 1274 dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err);
1275 goto clean_dev;
9603b61d
JM		 1276 }
1277
ac6ea6e8
EC		 1278 err = mlx5_health_init(dev);
1279 if (err) {
1280 dev_err(&pdev->dev, "mlx5_health_init failed with error code %d\n", err);
1281 goto close_pci;
1282 }
1283
a31208b1 1284 err = mlx5_load_one(dev, priv);
9603b61d 1285 if (err) {
a31208b1 1286 dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err);
ac6ea6e8 1287 goto clean_health;
9603b61d
JM		 1288 }
1289
1290 return 0;
1291
ac6ea6e8
EC		 1292clean_health:
1293 mlx5_health_cleanup(dev);
a31208b1
MD		 1294close_pci:
1295 mlx5_pci_close(dev, priv);
1296clean_dev:
1297 pci_set_drvdata(pdev, NULL);
9603b61d 1298 kfree(dev);
a31208b1 1299
9603b61d
JM		 1300 return err;
1301}
a31208b1 1302
9603b61d
JM		 1303static void remove_one(struct pci_dev *pdev)
1304{
1305 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
a31208b1 1306 struct mlx5_priv *priv = &dev->priv;
9603b61d 1307
a31208b1
MD		 1308 if (mlx5_unload_one(dev, priv)) {
1309 dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n");
ac6ea6e8 1310 mlx5_health_cleanup(dev);
a31208b1
MD		 1311 return;
1312 }
ac6ea6e8 1313 mlx5_health_cleanup(dev);
a31208b1
MD		 1314 mlx5_pci_close(dev, priv);
1315 pci_set_drvdata(pdev, NULL);
9603b61d
JM		 1316 kfree(dev);
1317}
1318
89d44f0a
MD		 1319static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
1320 pci_channel_state_t state)
1321{
1322 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1323 struct mlx5_priv *priv = &dev->priv;
1324
1325 dev_info(&pdev->dev, "%s was called\n", __func__);
1326 mlx5_enter_error_state(dev);
1327 mlx5_unload_one(dev, priv);
1328 mlx5_pci_disable_device(dev);
1329 return state == pci_channel_io_perm_failure ?
1330 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
1331}
1332
1333static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev)
1334{
1335 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1336 int err = 0;
1337
1338 dev_info(&pdev->dev, "%s was called\n", __func__);
1339
1340 err = mlx5_pci_enable_device(dev);
1341 if (err) {
1342 dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n"
1343 , __func__, err);
1344 return PCI_ERS_RESULT_DISCONNECT;
1345 }
1346 pci_set_master(pdev);
1347 pci_set_power_state(pdev, PCI_D0);
1348 pci_restore_state(pdev);
1349
1350 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
1351}
1352
1353void mlx5_disable_device(struct mlx5_core_dev *dev)
1354{
1355 mlx5_pci_err_detected(dev->pdev, 0);
1356}
1357
1358/* wait for the device to show vital signs. For now we check
1359 * that we can read the device ID and that the health buffer
1360 * shows a non zero value which is different than 0xffffffff
1361 */
1362static void wait_vital(struct pci_dev *pdev)
1363{
1364 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1365 struct mlx5_core_health *health = &dev->priv.health;
1366 const int niter = 100;
1367 u32 count;
1368 u16 did;
1369 int i;
1370
1371 /* Wait for firmware to be ready after reset */
1372 msleep(1000);
1373 for (i = 0; i < niter; i++) {
1374 if (pci_read_config_word(pdev, 2, &did)) {
1375 dev_warn(&pdev->dev, "failed reading config word\n");
1376 break;
1377 }
1378 if (did == pdev->device) {
1379 dev_info(&pdev->dev, "device ID correctly read after %d iterations\n", i);
1380 break;
1381 }
1382 msleep(50);
1383 }
1384 if (i == niter)
1385 dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__);
1386
1387 for (i = 0; i < niter; i++) {
1388 count = ioread32be(health->health_counter);
1389 if (count && count != 0xffffffff) {
1390 dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i);
1391 break;
1392 }
1393 msleep(50);
1394 }
1395
1396 if (i == niter)
1397 dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__);
1398}
1399
1400static void mlx5_pci_resume(struct pci_dev *pdev)
1401{
1402 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1403 struct mlx5_priv *priv = &dev->priv;
1404 int err;
1405
1406 dev_info(&pdev->dev, "%s was called\n", __func__);
1407
1408 pci_save_state(pdev);
1409 wait_vital(pdev);
1410
1411 err = mlx5_load_one(dev, priv);
1412 if (err)
1413 dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n"
1414 , __func__, err);
1415 else
1416 dev_info(&pdev->dev, "%s: device recovered\n", __func__);
1417}
1418
1419static const struct pci_error_handlers mlx5_err_handler = {
1420 .error_detected = mlx5_pci_err_detected,
1421 .slot_reset = mlx5_pci_slot_reset,
1422 .resume = mlx5_pci_resume
1423};
1424
9603b61d 1425static const struct pci_device_id mlx5_core_pci_table[] = {
fc50db98
EC		 1426 { PCI_VDEVICE(MELLANOX, 0x1011) }, /* Connect-IB */
1427 { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */
1428 { PCI_VDEVICE(MELLANOX, 0x1013) }, /* ConnectX-4 */
1429 { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */
1430 { PCI_VDEVICE(MELLANOX, 0x1015) }, /* ConnectX-4LX */
1431 { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */
9603b61d
JM		 1432 { 0, }
1433};
1434
1435MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
1436
1437static struct pci_driver mlx5_core_driver = {
1438 .name = DRIVER_NAME,
1439 .id_table = mlx5_core_pci_table,
1440 .probe = init_one,
89d44f0a 1441 .remove = remove_one,
fc50db98
EC		 1442 .err_handler = &mlx5_err_handler,
1443 .sriov_configure = mlx5_core_sriov_configure,
9603b61d 1444};
e126ba97
EC		 1445
1446static int __init init(void)
1447{
1448 int err;
1449
1450 mlx5_register_debugfs();
e126ba97 1451
9603b61d
JM		 1452 err = pci_register_driver(&mlx5_core_driver);
1453 if (err)
ac6ea6e8 1454 goto err_debug;
9603b61d 1455
f62b8bb8
AV		 1456#ifdef CONFIG_MLX5_CORE_EN
1457 mlx5e_init();
1458#endif
1459
e126ba97
EC		 1460 return 0;
1461
e126ba97
EC		 1462err_debug:
1463 mlx5_unregister_debugfs();
1464 return err;
1465}
1466
1467static void __exit cleanup(void)
1468{
f62b8bb8
AV		 1469#ifdef CONFIG_MLX5_CORE_EN
1470 mlx5e_cleanup();
1471#endif
9603b61d 1472 pci_unregister_driver(&mlx5_core_driver);
e126ba97
EC		 1473 mlx5_unregister_debugfs();
1474}
1475
1476module_init(init);
1477module_exit(cleanup);
Arne's tree of linux kernel.