]>
Commit | Line | Data |
---|---|---|
413d3347 MZ |
1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* | |
3 | * Copyright (c) 2019 Mellanox Technologies. All rights reserved. | |
4 | */ | |
5 | #include <rdma/ib_verbs.h> | |
6 | #include <rdma/rdma_counter.h> | |
7 | ||
8 | #include "core_priv.h" | |
9 | #include "restrack.h" | |
10 | ||
11 | #define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE) | |
12 | ||
13 | static int __counter_set_mode(struct rdma_counter_mode *curr, | |
14 | enum rdma_nl_counter_mode new_mode, | |
15 | enum rdma_nl_counter_mask new_mask) | |
16 | { | |
17 | if ((new_mode == RDMA_COUNTER_MODE_AUTO) && | |
18 | ((new_mask & (~ALL_AUTO_MODE_MASKS)) || | |
19 | (curr->mode != RDMA_COUNTER_MODE_NONE))) | |
20 | return -EINVAL; | |
21 | ||
22 | curr->mode = new_mode; | |
23 | curr->mask = new_mask; | |
24 | return 0; | |
25 | } | |
26 | ||
27 | /** | |
28 | * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode | |
29 | * | |
30 | * When @on is true, the @mask must be set | |
31 | */ | |
32 | int rdma_counter_set_auto_mode(struct ib_device *dev, u8 port, | |
33 | bool on, enum rdma_nl_counter_mask mask) | |
34 | { | |
35 | struct rdma_port_counter *port_counter; | |
36 | int ret; | |
37 | ||
38 | port_counter = &dev->port_data[port].port_counter; | |
39 | mutex_lock(&port_counter->lock); | |
40 | if (on) { | |
41 | ret = __counter_set_mode(&port_counter->mode, | |
42 | RDMA_COUNTER_MODE_AUTO, mask); | |
43 | } else { | |
44 | if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO) { | |
45 | ret = -EINVAL; | |
46 | goto out; | |
47 | } | |
48 | ret = __counter_set_mode(&port_counter->mode, | |
49 | RDMA_COUNTER_MODE_NONE, 0); | |
50 | } | |
51 | ||
52 | out: | |
53 | mutex_unlock(&port_counter->lock); | |
54 | return ret; | |
55 | } | |
56 | ||
99fa331d MZ |
57 | static struct rdma_counter *rdma_counter_alloc(struct ib_device *dev, u8 port, |
58 | enum rdma_nl_counter_mode mode) | |
59 | { | |
60 | struct rdma_counter *counter; | |
61 | ||
c4ffee7c | 62 | if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats) |
99fa331d MZ |
63 | return NULL; |
64 | ||
65 | counter = kzalloc(sizeof(*counter), GFP_KERNEL); | |
66 | if (!counter) | |
67 | return NULL; | |
68 | ||
69 | counter->device = dev; | |
70 | counter->port = port; | |
71 | counter->res.type = RDMA_RESTRACK_COUNTER; | |
c4ffee7c MZ |
72 | counter->stats = dev->ops.counter_alloc_stats(counter); |
73 | if (!counter->stats) | |
74 | goto err_stats; | |
75 | ||
99fa331d MZ |
76 | counter->mode.mode = mode; |
77 | kref_init(&counter->kref); | |
78 | mutex_init(&counter->lock); | |
79 | ||
80 | return counter; | |
c4ffee7c MZ |
81 | |
82 | err_stats: | |
83 | kfree(counter); | |
84 | return NULL; | |
99fa331d MZ |
85 | } |
86 | ||
87 | static void rdma_counter_free(struct rdma_counter *counter) | |
88 | { | |
89 | rdma_restrack_del(&counter->res); | |
c4ffee7c | 90 | kfree(counter->stats); |
99fa331d MZ |
91 | kfree(counter); |
92 | } | |
93 | ||
94 | static void auto_mode_init_counter(struct rdma_counter *counter, | |
95 | const struct ib_qp *qp, | |
96 | enum rdma_nl_counter_mask new_mask) | |
97 | { | |
98 | struct auto_mode_param *param = &counter->mode.param; | |
99 | ||
100 | counter->mode.mode = RDMA_COUNTER_MODE_AUTO; | |
101 | counter->mode.mask = new_mask; | |
102 | ||
103 | if (new_mask & RDMA_COUNTER_MASK_QP_TYPE) | |
104 | param->qp_type = qp->qp_type; | |
105 | } | |
106 | ||
107 | static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter, | |
108 | enum rdma_nl_counter_mask auto_mask) | |
109 | { | |
110 | struct auto_mode_param *param = &counter->mode.param; | |
111 | bool match = true; | |
112 | ||
113 | if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) | |
114 | return false; | |
115 | ||
116 | /* Ensure that counter belong to right PID */ | |
117 | if (!rdma_is_kernel_res(&counter->res) && | |
118 | !rdma_is_kernel_res(&qp->res) && | |
119 | (task_pid_vnr(counter->res.task) != current->pid)) | |
120 | return false; | |
121 | ||
122 | if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE) | |
123 | match &= (param->qp_type == qp->qp_type); | |
124 | ||
125 | return match; | |
126 | } | |
127 | ||
128 | static int __rdma_counter_bind_qp(struct rdma_counter *counter, | |
129 | struct ib_qp *qp) | |
130 | { | |
131 | int ret; | |
132 | ||
133 | if (qp->counter) | |
134 | return -EINVAL; | |
135 | ||
136 | if (!qp->device->ops.counter_bind_qp) | |
137 | return -EOPNOTSUPP; | |
138 | ||
139 | mutex_lock(&counter->lock); | |
140 | ret = qp->device->ops.counter_bind_qp(counter, qp); | |
141 | mutex_unlock(&counter->lock); | |
142 | ||
143 | return ret; | |
144 | } | |
145 | ||
146 | static int __rdma_counter_unbind_qp(struct ib_qp *qp) | |
147 | { | |
148 | struct rdma_counter *counter = qp->counter; | |
149 | int ret; | |
150 | ||
151 | if (!qp->device->ops.counter_unbind_qp) | |
152 | return -EOPNOTSUPP; | |
153 | ||
154 | mutex_lock(&counter->lock); | |
155 | ret = qp->device->ops.counter_unbind_qp(qp); | |
156 | mutex_unlock(&counter->lock); | |
157 | ||
158 | return ret; | |
159 | } | |
160 | ||
f34a55e4 MZ |
161 | static void counter_history_stat_update(const struct rdma_counter *counter) |
162 | { | |
163 | struct ib_device *dev = counter->device; | |
164 | struct rdma_port_counter *port_counter; | |
165 | int i; | |
166 | ||
167 | port_counter = &dev->port_data[counter->port].port_counter; | |
168 | if (!port_counter->hstats) | |
169 | return; | |
170 | ||
171 | for (i = 0; i < counter->stats->num_counters; i++) | |
172 | port_counter->hstats->value[i] += counter->stats->value[i]; | |
173 | } | |
174 | ||
99fa331d MZ |
175 | /** |
176 | * rdma_get_counter_auto_mode - Find the counter that @qp should be bound | |
177 | * with in auto mode | |
178 | * | |
179 | * Return: The counter (with ref-count increased) if found | |
180 | */ | |
181 | static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp, | |
182 | u8 port) | |
183 | { | |
184 | struct rdma_port_counter *port_counter; | |
185 | struct rdma_counter *counter = NULL; | |
186 | struct ib_device *dev = qp->device; | |
187 | struct rdma_restrack_entry *res; | |
188 | struct rdma_restrack_root *rt; | |
189 | unsigned long id = 0; | |
190 | ||
191 | port_counter = &dev->port_data[port].port_counter; | |
192 | rt = &dev->res[RDMA_RESTRACK_COUNTER]; | |
193 | xa_lock(&rt->xa); | |
194 | xa_for_each(&rt->xa, id, res) { | |
195 | if (!rdma_is_visible_in_pid_ns(res)) | |
196 | continue; | |
197 | ||
198 | counter = container_of(res, struct rdma_counter, res); | |
199 | if ((counter->device != qp->device) || (counter->port != port)) | |
200 | goto next; | |
201 | ||
202 | if (auto_mode_match(qp, counter, port_counter->mode.mask)) | |
203 | break; | |
204 | next: | |
205 | counter = NULL; | |
206 | } | |
207 | ||
208 | if (counter && !kref_get_unless_zero(&counter->kref)) | |
209 | counter = NULL; | |
210 | ||
211 | xa_unlock(&rt->xa); | |
212 | return counter; | |
213 | } | |
214 | ||
215 | static void rdma_counter_res_add(struct rdma_counter *counter, | |
216 | struct ib_qp *qp) | |
217 | { | |
218 | if (rdma_is_kernel_res(&qp->res)) { | |
219 | rdma_restrack_set_task(&counter->res, qp->res.kern_name); | |
220 | rdma_restrack_kadd(&counter->res); | |
221 | } else { | |
222 | rdma_restrack_attach_task(&counter->res, qp->res.task); | |
223 | rdma_restrack_uadd(&counter->res); | |
224 | } | |
225 | } | |
226 | ||
227 | static void counter_release(struct kref *kref) | |
228 | { | |
229 | struct rdma_counter *counter; | |
230 | ||
231 | counter = container_of(kref, struct rdma_counter, kref); | |
f34a55e4 | 232 | counter_history_stat_update(counter); |
99fa331d MZ |
233 | counter->device->ops.counter_dealloc(counter); |
234 | rdma_counter_free(counter); | |
235 | } | |
236 | ||
237 | /** | |
238 | * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on | |
239 | * the auto-mode rule | |
240 | */ | |
241 | int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port) | |
242 | { | |
243 | struct rdma_port_counter *port_counter; | |
244 | struct ib_device *dev = qp->device; | |
245 | struct rdma_counter *counter; | |
246 | int ret; | |
247 | ||
248 | if (!rdma_is_port_valid(dev, port)) | |
249 | return -EINVAL; | |
250 | ||
251 | port_counter = &dev->port_data[port].port_counter; | |
252 | if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO) | |
253 | return 0; | |
254 | ||
255 | counter = rdma_get_counter_auto_mode(qp, port); | |
256 | if (counter) { | |
257 | ret = __rdma_counter_bind_qp(counter, qp); | |
258 | if (ret) { | |
259 | kref_put(&counter->kref, counter_release); | |
260 | return ret; | |
261 | } | |
262 | } else { | |
263 | counter = rdma_counter_alloc(dev, port, RDMA_COUNTER_MODE_AUTO); | |
264 | if (!counter) | |
265 | return -ENOMEM; | |
266 | ||
267 | auto_mode_init_counter(counter, qp, port_counter->mode.mask); | |
268 | ||
269 | ret = __rdma_counter_bind_qp(counter, qp); | |
270 | if (ret) { | |
271 | rdma_counter_free(counter); | |
272 | return ret; | |
273 | } | |
274 | ||
275 | rdma_counter_res_add(counter, qp); | |
276 | } | |
277 | ||
278 | return 0; | |
279 | } | |
280 | ||
281 | /** | |
282 | * rdma_counter_unbind_qp - Unbind a qp from a counter | |
283 | * @force: | |
284 | * true - Decrease the counter ref-count anyway (e.g., qp destroy) | |
285 | */ | |
286 | int rdma_counter_unbind_qp(struct ib_qp *qp, bool force) | |
287 | { | |
288 | struct rdma_counter *counter = qp->counter; | |
289 | int ret; | |
290 | ||
291 | if (!counter) | |
292 | return -EINVAL; | |
293 | ||
294 | ret = __rdma_counter_unbind_qp(qp); | |
295 | if (ret && !force) | |
296 | return ret; | |
297 | ||
298 | kref_put(&counter->kref, counter_release); | |
299 | return 0; | |
300 | } | |
301 | ||
c4ffee7c MZ |
302 | int rdma_counter_query_stats(struct rdma_counter *counter) |
303 | { | |
304 | struct ib_device *dev = counter->device; | |
305 | int ret; | |
306 | ||
307 | if (!dev->ops.counter_update_stats) | |
308 | return -EINVAL; | |
309 | ||
310 | mutex_lock(&counter->lock); | |
311 | ret = dev->ops.counter_update_stats(counter); | |
312 | mutex_unlock(&counter->lock); | |
313 | ||
314 | return ret; | |
315 | } | |
316 | ||
f34a55e4 MZ |
317 | static u64 get_running_counters_hwstat_sum(struct ib_device *dev, |
318 | u8 port, u32 index) | |
319 | { | |
320 | struct rdma_restrack_entry *res; | |
321 | struct rdma_restrack_root *rt; | |
322 | struct rdma_counter *counter; | |
323 | unsigned long id = 0; | |
324 | u64 sum = 0; | |
325 | ||
326 | rt = &dev->res[RDMA_RESTRACK_COUNTER]; | |
327 | xa_lock(&rt->xa); | |
328 | xa_for_each(&rt->xa, id, res) { | |
329 | if (!rdma_restrack_get(res)) | |
330 | continue; | |
331 | ||
332 | xa_unlock(&rt->xa); | |
333 | ||
334 | counter = container_of(res, struct rdma_counter, res); | |
335 | if ((counter->device != dev) || (counter->port != port) || | |
336 | rdma_counter_query_stats(counter)) | |
337 | goto next; | |
338 | ||
339 | sum += counter->stats->value[index]; | |
340 | ||
341 | next: | |
342 | xa_lock(&rt->xa); | |
343 | rdma_restrack_put(res); | |
344 | } | |
345 | ||
346 | xa_unlock(&rt->xa); | |
347 | return sum; | |
348 | } | |
349 | ||
350 | /** | |
351 | * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a | |
352 | * specific port, including the running ones and history data | |
353 | */ | |
354 | u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u8 port, u32 index) | |
355 | { | |
356 | struct rdma_port_counter *port_counter; | |
357 | u64 sum; | |
358 | ||
359 | port_counter = &dev->port_data[port].port_counter; | |
360 | sum = get_running_counters_hwstat_sum(dev, port, index); | |
361 | sum += port_counter->hstats->value[index]; | |
362 | ||
363 | return sum; | |
364 | } | |
365 | ||
413d3347 MZ |
366 | void rdma_counter_init(struct ib_device *dev) |
367 | { | |
368 | struct rdma_port_counter *port_counter; | |
369 | u32 port; | |
370 | ||
371 | if (!dev->ops.alloc_hw_stats || !dev->port_data) | |
372 | return; | |
373 | ||
374 | rdma_for_each_port(dev, port) { | |
375 | port_counter = &dev->port_data[port].port_counter; | |
376 | port_counter->mode.mode = RDMA_COUNTER_MODE_NONE; | |
377 | mutex_init(&port_counter->lock); | |
f34a55e4 MZ |
378 | |
379 | port_counter->hstats = dev->ops.alloc_hw_stats(dev, port); | |
380 | if (!port_counter->hstats) | |
381 | goto fail; | |
413d3347 | 382 | } |
f34a55e4 MZ |
383 | |
384 | return; | |
385 | ||
386 | fail: | |
387 | rdma_for_each_port(dev, port) { | |
388 | port_counter = &dev->port_data[port].port_counter; | |
389 | kfree(port_counter->hstats); | |
390 | port_counter->hstats = NULL; | |
391 | } | |
392 | ||
393 | return; | |
413d3347 MZ |
394 | } |
395 | ||
396 | void rdma_counter_release(struct ib_device *dev) | |
397 | { | |
f34a55e4 MZ |
398 | struct rdma_port_counter *port_counter; |
399 | u32 port; | |
400 | ||
401 | if (!dev->ops.alloc_hw_stats) | |
402 | return; | |
403 | ||
404 | rdma_for_each_port(dev, port) { | |
405 | port_counter = &dev->port_data[port].port_counter; | |
406 | kfree(port_counter->hstats); | |
407 | } | |
413d3347 | 408 | } |