]>
Commit | Line | Data |
---|---|---|
937cd4b4 GKH |
1 | From 8ec07bf8a8b57d6c58927a16a0a22c0115cf2855 Mon Sep 17 00:00:00 2001 |
2 | From: Jack Morgenstein <jackm@dev.mellanox.co.il> | |
3 | Date: Mon, 12 Sep 2016 19:16:20 +0300 | |
4 | Subject: IB/mlx4: Use correct subnet-prefix in QP1 mads under SR-IOV | |
5 | ||
6 | From: Jack Morgenstein <jackm@dev.mellanox.co.il> | |
7 | ||
8 | commit 8ec07bf8a8b57d6c58927a16a0a22c0115cf2855 upstream. | |
9 | ||
10 | When sending QP1 MAD packets which use a GRH, the source GID | |
11 | (which consists of the 64-bit subnet prefix, and the 64 bit port GUID) | |
12 | must be included in the packet GRH. | |
13 | ||
14 | For SR-IOV, a GID cache is used, since the source GID needs to be the | |
15 | slave's source GID, and not the Hypervisor's GID. This cache also | |
16 | included a subnet_prefix. Unfortunately, the subnet_prefix field in | |
17 | the cache was never initialized (to the default subnet prefix 0xfe80::0). | |
18 | As a result, this field remained all zeroes. Therefore, when SR-IOV | |
19 | was active, all QP1 packets which included a GRH had a source GID | |
20 | subnet prefix of all-zeroes. | |
21 | ||
22 | However, the subnet-prefix should initially be 0xfe80::0 (the default | |
23 | subnet prefix). In addition, if OpenSM modifies a port's subnet prefix, | |
24 | the new subnet prefix must be used in the GRH when sending QP1 packets. | |
25 | To fix this we now initialize the subnet prefix in the SR-IOV GID cache | |
26 | to the default subnet prefix. We update the cached value if/when OpenSM | |
27 | modifies the port's subnet prefix. We take this cached value when sending | |
28 | QP1 packets when SR-IOV is active. | |
29 | ||
30 | Note that the value is stored as an atomic64. This eliminates any need | |
31 | for locking when the subnet prefix is being updated. | |
32 | ||
33 | Note also that we depend on the FW generating the "port management change" | |
34 | event for tracking subnet-prefix changes performed by OpenSM. If running | |
35 | early FW (before 2.9.4630), subnet prefix changes will not be tracked (but | |
36 | the default subnet prefix still will be stored in the cache; therefore | |
37 | users who do not modify the subnet prefix will not have a problem). | |
38 | IF there is a need for such tracking also for early FW, we will add that | |
39 | capability in a subsequent patch. | |
40 | ||
41 | Fixes: 1ffeb2eb8be9 ("IB/mlx4: SR-IOV IB context objects and proxy/tunnel SQP support") | |
42 | Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il> | |
43 | Signed-off-by: Leon Romanovsky <leon@kernel.org> | |
44 | Signed-off-by: Doug Ledford <dledford@redhat.com> | |
45 | Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> | |
46 | ||
47 | --- | |
48 | drivers/infiniband/hw/mlx4/mad.c | 23 +++++++++++++++++++++++ | |
49 | drivers/infiniband/hw/mlx4/mlx4_ib.h | 2 +- | |
50 | drivers/infiniband/hw/mlx4/qp.c | 5 +++-- | |
51 | 3 files changed, 27 insertions(+), 3 deletions(-) | |
52 | ||
53 | --- a/drivers/infiniband/hw/mlx4/mad.c | |
54 | +++ b/drivers/infiniband/hw/mlx4/mad.c | |
55 | @@ -1128,6 +1128,27 @@ void handle_port_mgmt_change_event(struc | |
56 | ||
57 | /* Generate GUID changed event */ | |
58 | if (changed_attr & MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK) { | |
59 | + if (mlx4_is_master(dev->dev)) { | |
60 | + union ib_gid gid; | |
61 | + int err = 0; | |
62 | + | |
63 | + if (!eqe->event.port_mgmt_change.params.port_info.gid_prefix) | |
64 | + err = __mlx4_ib_query_gid(&dev->ib_dev, port, 0, &gid, 1); | |
65 | + else | |
66 | + gid.global.subnet_prefix = | |
67 | + eqe->event.port_mgmt_change.params.port_info.gid_prefix; | |
68 | + if (err) { | |
69 | + pr_warn("Could not change QP1 subnet prefix for port %d: query_gid error (%d)\n", | |
70 | + port, err); | |
71 | + } else { | |
72 | + pr_debug("Changing QP1 subnet prefix for port %d. old=0x%llx. new=0x%llx\n", | |
73 | + port, | |
74 | + (u64)atomic64_read(&dev->sriov.demux[port - 1].subnet_prefix), | |
75 | + be64_to_cpu(gid.global.subnet_prefix)); | |
76 | + atomic64_set(&dev->sriov.demux[port - 1].subnet_prefix, | |
77 | + be64_to_cpu(gid.global.subnet_prefix)); | |
78 | + } | |
79 | + } | |
80 | mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE); | |
81 | /*if master, notify all slaves*/ | |
82 | if (mlx4_is_master(dev->dev)) | |
83 | @@ -2202,6 +2223,8 @@ int mlx4_ib_init_sriov(struct mlx4_ib_de | |
84 | if (err) | |
85 | goto demux_err; | |
86 | dev->sriov.demux[i].guid_cache[0] = gid.global.interface_id; | |
87 | + atomic64_set(&dev->sriov.demux[i].subnet_prefix, | |
88 | + be64_to_cpu(gid.global.subnet_prefix)); | |
89 | err = alloc_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1, | |
90 | &dev->sriov.sqps[i]); | |
91 | if (err) | |
92 | --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h | |
93 | +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h | |
94 | @@ -448,7 +448,7 @@ struct mlx4_ib_demux_ctx { | |
95 | struct workqueue_struct *wq; | |
96 | struct workqueue_struct *ud_wq; | |
97 | spinlock_t ud_lock; | |
98 | - __be64 subnet_prefix; | |
99 | + atomic64_t subnet_prefix; | |
100 | __be64 guid_cache[128]; | |
101 | struct mlx4_ib_dev *dev; | |
102 | /* the following lock protects both mcg_table and mcg_mgid0_list */ | |
103 | --- a/drivers/infiniband/hw/mlx4/qp.c | |
104 | +++ b/drivers/infiniband/hw/mlx4/qp.c | |
105 | @@ -2501,8 +2501,9 @@ static int build_mlx_header(struct mlx4_ | |
106 | * we must use our own cache | |
107 | */ | |
108 | sqp->ud_header.grh.source_gid.global.subnet_prefix = | |
109 | - to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. | |
110 | - subnet_prefix; | |
111 | + cpu_to_be64(atomic64_read(&(to_mdev(ib_dev)->sriov. | |
112 | + demux[sqp->qp.port - 1]. | |
113 | + subnet_prefix))); | |
114 | sqp->ud_header.grh.source_gid.global.interface_id = | |
115 | to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. | |
116 | guid_cache[ah->av.ib.gid_index]; |