2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #define CREATE_TRACE_POINTS
34 #include "fw_tracer.h"
35 #include "fw_tracer_tracepoint.h"
37 static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer
*tracer
)
39 u32
*string_db_base_address_out
= tracer
->str_db
.base_address_out
;
40 u32
*string_db_size_out
= tracer
->str_db
.size_out
;
41 struct mlx5_core_dev
*dev
= tracer
->dev
;
42 u32 out
[MLX5_ST_SZ_DW(mtrc_cap
)] = {0};
43 u32 in
[MLX5_ST_SZ_DW(mtrc_cap
)] = {0};
47 err
= mlx5_core_access_reg(dev
, in
, sizeof(in
), out
, sizeof(out
),
48 MLX5_REG_MTRC_CAP
, 0, 0);
50 mlx5_core_warn(dev
, "FWTracer: Error reading tracer caps %d\n",
55 if (!MLX5_GET(mtrc_cap
, out
, trace_to_memory
)) {
56 mlx5_core_dbg(dev
, "FWTracer: Device does not support logging traces to memory\n");
60 tracer
->trc_ver
= MLX5_GET(mtrc_cap
, out
, trc_ver
);
61 tracer
->str_db
.first_string_trace
=
62 MLX5_GET(mtrc_cap
, out
, first_string_trace
);
63 tracer
->str_db
.num_string_trace
=
64 MLX5_GET(mtrc_cap
, out
, num_string_trace
);
65 tracer
->str_db
.num_string_db
= MLX5_GET(mtrc_cap
, out
, num_string_db
);
66 tracer
->owner
= !!MLX5_GET(mtrc_cap
, out
, trace_owner
);
68 for (i
= 0; i
< tracer
->str_db
.num_string_db
; i
++) {
69 mtrc_cap_sp
= MLX5_ADDR_OF(mtrc_cap
, out
, string_db_param
[i
]);
70 string_db_base_address_out
[i
] = MLX5_GET(mtrc_string_db_param
,
72 string_db_base_address
);
73 string_db_size_out
[i
] = MLX5_GET(mtrc_string_db_param
,
74 mtrc_cap_sp
, string_db_size
);
80 static int mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer
*tracer
,
81 u32
*out
, u32 out_size
,
84 struct mlx5_core_dev
*dev
= tracer
->dev
;
85 u32 in
[MLX5_ST_SZ_DW(mtrc_cap
)] = {0};
87 MLX5_SET(mtrc_cap
, in
, trace_owner
, trace_owner
);
89 return mlx5_core_access_reg(dev
, in
, sizeof(in
), out
, out_size
,
90 MLX5_REG_MTRC_CAP
, 0, 1);
93 static int mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer
*tracer
)
95 struct mlx5_core_dev
*dev
= tracer
->dev
;
96 u32 out
[MLX5_ST_SZ_DW(mtrc_cap
)] = {0};
99 err
= mlx5_set_mtrc_caps_trace_owner(tracer
, out
, sizeof(out
),
100 MLX5_FW_TRACER_ACQUIRE_OWNERSHIP
);
102 mlx5_core_warn(dev
, "FWTracer: Acquire tracer ownership failed %d\n",
107 tracer
->owner
= !!MLX5_GET(mtrc_cap
, out
, trace_owner
);
115 static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer
*tracer
)
117 u32 out
[MLX5_ST_SZ_DW(mtrc_cap
)] = {0};
119 mlx5_set_mtrc_caps_trace_owner(tracer
, out
, sizeof(out
),
120 MLX5_FW_TRACER_RELEASE_OWNERSHIP
);
121 tracer
->owner
= false;
124 static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer
*tracer
)
126 struct mlx5_core_dev
*dev
= tracer
->dev
;
127 struct device
*ddev
= &dev
->pdev
->dev
;
133 tracer
->buff
.size
= TRACE_BUFFER_SIZE_BYTE
;
135 gfp
= GFP_KERNEL
| __GFP_ZERO
;
136 buff
= (void *)__get_free_pages(gfp
,
137 get_order(tracer
->buff
.size
));
140 mlx5_core_warn(dev
, "FWTracer: Failed to allocate pages, %d\n", err
);
143 tracer
->buff
.log_buf
= buff
;
145 dma
= dma_map_single(ddev
, buff
, tracer
->buff
.size
, DMA_FROM_DEVICE
);
146 if (dma_mapping_error(ddev
, dma
)) {
147 mlx5_core_warn(dev
, "FWTracer: Unable to map DMA: %d\n",
148 dma_mapping_error(ddev
, dma
));
152 tracer
->buff
.dma
= dma
;
157 free_pages((unsigned long)tracer
->buff
.log_buf
, get_order(tracer
->buff
.size
));
162 static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer
*tracer
)
164 struct mlx5_core_dev
*dev
= tracer
->dev
;
165 struct device
*ddev
= &dev
->pdev
->dev
;
167 if (!tracer
->buff
.log_buf
)
170 dma_unmap_single(ddev
, tracer
->buff
.dma
, tracer
->buff
.size
, DMA_FROM_DEVICE
);
171 free_pages((unsigned long)tracer
->buff
.log_buf
, get_order(tracer
->buff
.size
));
174 static int mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer
*tracer
)
176 struct mlx5_core_dev
*dev
= tracer
->dev
;
182 inlen
= MLX5_ST_SZ_BYTES(create_mkey_in
) +
183 sizeof(*mtt
) * round_up(TRACER_BUFFER_PAGE_NUM
, 2);
185 in
= kvzalloc(inlen
, GFP_KERNEL
);
189 MLX5_SET(create_mkey_in
, in
, translations_octword_actual_size
,
190 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM
, 2));
191 mtt
= (__be64
*)MLX5_ADDR_OF(create_mkey_in
, in
, klm_pas_mtt
);
192 for (i
= 0 ; i
< TRACER_BUFFER_PAGE_NUM
; i
++)
193 mtt
[i
] = cpu_to_be64(tracer
->buff
.dma
+ i
* PAGE_SIZE
);
195 mkc
= MLX5_ADDR_OF(create_mkey_in
, in
, memory_key_mkey_entry
);
196 MLX5_SET(mkc
, mkc
, access_mode_1_0
, MLX5_MKC_ACCESS_MODE_MTT
);
197 MLX5_SET(mkc
, mkc
, lr
, 1);
198 MLX5_SET(mkc
, mkc
, lw
, 1);
199 MLX5_SET(mkc
, mkc
, pd
, tracer
->buff
.pdn
);
200 MLX5_SET(mkc
, mkc
, bsf_octword_size
, 0);
201 MLX5_SET(mkc
, mkc
, qpn
, 0xffffff);
202 MLX5_SET(mkc
, mkc
, log_page_size
, PAGE_SHIFT
);
203 MLX5_SET(mkc
, mkc
, translations_octword_size
,
204 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM
, 2));
205 MLX5_SET64(mkc
, mkc
, start_addr
, tracer
->buff
.dma
);
206 MLX5_SET64(mkc
, mkc
, len
, tracer
->buff
.size
);
207 err
= mlx5_core_create_mkey(dev
, &tracer
->buff
.mkey
, in
, inlen
);
209 mlx5_core_warn(dev
, "FWTracer: Failed to create mkey, %d\n", err
);
216 static void mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer
*tracer
)
218 u32 num_string_db
= tracer
->str_db
.num_string_db
;
221 for (i
= 0; i
< num_string_db
; i
++) {
222 kfree(tracer
->str_db
.buffer
[i
]);
223 tracer
->str_db
.buffer
[i
] = NULL
;
227 static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer
*tracer
)
229 u32
*string_db_size_out
= tracer
->str_db
.size_out
;
230 u32 num_string_db
= tracer
->str_db
.num_string_db
;
233 for (i
= 0; i
< num_string_db
; i
++) {
234 tracer
->str_db
.buffer
[i
] = kzalloc(string_db_size_out
[i
], GFP_KERNEL
);
235 if (!tracer
->str_db
.buffer
[i
])
236 goto free_strings_db
;
242 mlx5_fw_tracer_free_strings_db(tracer
);
247 mlx5_fw_tracer_init_saved_traces_array(struct mlx5_fw_tracer
*tracer
)
249 tracer
->st_arr
.saved_traces_index
= 0;
250 mutex_init(&tracer
->st_arr
.lock
);
254 mlx5_fw_tracer_clean_saved_traces_array(struct mlx5_fw_tracer
*tracer
)
256 mutex_destroy(&tracer
->st_arr
.lock
);
259 static void mlx5_tracer_read_strings_db(struct work_struct
*work
)
261 struct mlx5_fw_tracer
*tracer
= container_of(work
, struct mlx5_fw_tracer
,
262 read_fw_strings_work
);
263 u32 num_of_reads
, num_string_db
= tracer
->str_db
.num_string_db
;
264 struct mlx5_core_dev
*dev
= tracer
->dev
;
265 u32 in
[MLX5_ST_SZ_DW(mtrc_cap
)] = {0};
266 u32 leftovers
, offset
;
271 outlen
= MLX5_ST_SZ_BYTES(mtrc_stdb
) + STRINGS_DB_READ_SIZE_BYTES
;
272 out
= kzalloc(outlen
, GFP_KERNEL
);
278 for (i
= 0; i
< num_string_db
; i
++) {
280 MLX5_SET(mtrc_stdb
, in
, string_db_index
, i
);
281 num_of_reads
= tracer
->str_db
.size_out
[i
] /
282 STRINGS_DB_READ_SIZE_BYTES
;
283 leftovers
= (tracer
->str_db
.size_out
[i
] %
284 STRINGS_DB_READ_SIZE_BYTES
) /
285 STRINGS_DB_LEFTOVER_SIZE_BYTES
;
287 MLX5_SET(mtrc_stdb
, in
, read_size
, STRINGS_DB_READ_SIZE_BYTES
);
288 for (j
= 0; j
< num_of_reads
; j
++) {
289 MLX5_SET(mtrc_stdb
, in
, start_offset
, offset
);
291 err
= mlx5_core_access_reg(dev
, in
, sizeof(in
), out
,
292 outlen
, MLX5_REG_MTRC_STDB
,
295 mlx5_core_dbg(dev
, "FWTracer: Failed to read strings DB %d\n",
300 out_value
= MLX5_ADDR_OF(mtrc_stdb
, out
, string_db_data
);
301 memcpy(tracer
->str_db
.buffer
[i
] + offset
, out_value
,
302 STRINGS_DB_READ_SIZE_BYTES
);
303 offset
+= STRINGS_DB_READ_SIZE_BYTES
;
306 /* Strings database is aligned to 64, need to read leftovers*/
307 MLX5_SET(mtrc_stdb
, in
, read_size
,
308 STRINGS_DB_LEFTOVER_SIZE_BYTES
);
309 for (j
= 0; j
< leftovers
; j
++) {
310 MLX5_SET(mtrc_stdb
, in
, start_offset
, offset
);
312 err
= mlx5_core_access_reg(dev
, in
, sizeof(in
), out
,
313 outlen
, MLX5_REG_MTRC_STDB
,
316 mlx5_core_dbg(dev
, "FWTracer: Failed to read strings DB %d\n",
321 out_value
= MLX5_ADDR_OF(mtrc_stdb
, out
, string_db_data
);
322 memcpy(tracer
->str_db
.buffer
[i
] + offset
, out_value
,
323 STRINGS_DB_LEFTOVER_SIZE_BYTES
);
324 offset
+= STRINGS_DB_LEFTOVER_SIZE_BYTES
;
328 tracer
->str_db
.loaded
= true;
336 static void mlx5_fw_tracer_arm(struct mlx5_core_dev
*dev
)
338 u32 out
[MLX5_ST_SZ_DW(mtrc_ctrl
)] = {0};
339 u32 in
[MLX5_ST_SZ_DW(mtrc_ctrl
)] = {0};
342 MLX5_SET(mtrc_ctrl
, in
, arm_event
, 1);
344 err
= mlx5_core_access_reg(dev
, in
, sizeof(in
), out
, sizeof(out
),
345 MLX5_REG_MTRC_CTRL
, 0, 1);
347 mlx5_core_warn(dev
, "FWTracer: Failed to arm tracer event %d\n", err
);
350 static const char *VAL_PARM
= "%llx";
351 static const char *REPLACE_64_VAL_PARM
= "%x%x";
352 static const char *PARAM_CHAR
= "%";
354 static int mlx5_tracer_message_hash(u32 message_id
)
356 return jhash_1word(message_id
, 0) & (MESSAGE_HASH_SIZE
- 1);
359 static struct tracer_string_format
*mlx5_tracer_message_insert(struct mlx5_fw_tracer
*tracer
,
360 struct tracer_event
*tracer_event
)
362 struct hlist_head
*head
=
363 &tracer
->hash
[mlx5_tracer_message_hash(tracer_event
->string_event
.tmsn
)];
364 struct tracer_string_format
*cur_string
;
366 cur_string
= kzalloc(sizeof(*cur_string
), GFP_KERNEL
);
370 hlist_add_head(&cur_string
->hlist
, head
);
375 static struct tracer_string_format
*mlx5_tracer_get_string(struct mlx5_fw_tracer
*tracer
,
376 struct tracer_event
*tracer_event
)
378 struct tracer_string_format
*cur_string
;
382 str_ptr
= tracer_event
->string_event
.string_param
;
384 for (i
= 0; i
< tracer
->str_db
.num_string_db
; i
++) {
385 if (str_ptr
> tracer
->str_db
.base_address_out
[i
] &&
386 str_ptr
< tracer
->str_db
.base_address_out
[i
] +
387 tracer
->str_db
.size_out
[i
]) {
388 offset
= str_ptr
- tracer
->str_db
.base_address_out
[i
];
389 /* add it to the hash */
390 cur_string
= mlx5_tracer_message_insert(tracer
, tracer_event
);
393 cur_string
->string
= (char *)(tracer
->str_db
.buffer
[i
] +
402 static void mlx5_tracer_clean_message(struct tracer_string_format
*str_frmt
)
404 hlist_del(&str_frmt
->hlist
);
408 static int mlx5_tracer_get_num_of_params(char *str
)
410 char *substr
, *pstr
= str
;
411 int num_of_params
= 0;
413 /* replace %llx with %x%x */
414 substr
= strstr(pstr
, VAL_PARM
);
416 memcpy(substr
, REPLACE_64_VAL_PARM
, 4);
418 substr
= strstr(pstr
, VAL_PARM
);
421 /* count all the % characters */
422 substr
= strstr(str
, PARAM_CHAR
);
426 substr
= strstr(str
, PARAM_CHAR
);
429 return num_of_params
;
432 static struct tracer_string_format
*mlx5_tracer_message_find(struct hlist_head
*head
,
433 u8 event_id
, u32 tmsn
)
435 struct tracer_string_format
*message
;
437 hlist_for_each_entry(message
, head
, hlist
)
438 if (message
->event_id
== event_id
&& message
->tmsn
== tmsn
)
444 static struct tracer_string_format
*mlx5_tracer_message_get(struct mlx5_fw_tracer
*tracer
,
445 struct tracer_event
*tracer_event
)
447 struct hlist_head
*head
=
448 &tracer
->hash
[mlx5_tracer_message_hash(tracer_event
->string_event
.tmsn
)];
450 return mlx5_tracer_message_find(head
, tracer_event
->event_id
, tracer_event
->string_event
.tmsn
);
453 static void poll_trace(struct mlx5_fw_tracer
*tracer
,
454 struct tracer_event
*tracer_event
, u64
*trace
)
456 u32 timestamp_low
, timestamp_mid
, timestamp_high
, urts
;
458 tracer_event
->event_id
= MLX5_GET(tracer_event
, trace
, event_id
);
459 tracer_event
->lost_event
= MLX5_GET(tracer_event
, trace
, lost
);
461 switch (tracer_event
->event_id
) {
462 case TRACER_EVENT_TYPE_TIMESTAMP
:
463 tracer_event
->type
= TRACER_EVENT_TYPE_TIMESTAMP
;
464 urts
= MLX5_GET(tracer_timestamp_event
, trace
, urts
);
465 if (tracer
->trc_ver
== 0)
466 tracer_event
->timestamp_event
.unreliable
= !!(urts
>> 2);
468 tracer_event
->timestamp_event
.unreliable
= !!(urts
& 1);
470 timestamp_low
= MLX5_GET(tracer_timestamp_event
,
471 trace
, timestamp7_0
);
472 timestamp_mid
= MLX5_GET(tracer_timestamp_event
,
473 trace
, timestamp39_8
);
474 timestamp_high
= MLX5_GET(tracer_timestamp_event
,
475 trace
, timestamp52_40
);
477 tracer_event
->timestamp_event
.timestamp
=
478 ((u64
)timestamp_high
<< 40) |
479 ((u64
)timestamp_mid
<< 8) |
483 if (tracer_event
->event_id
>= tracer
->str_db
.first_string_trace
||
484 tracer_event
->event_id
<= tracer
->str_db
.first_string_trace
+
485 tracer
->str_db
.num_string_trace
) {
486 tracer_event
->type
= TRACER_EVENT_TYPE_STRING
;
487 tracer_event
->string_event
.timestamp
=
488 MLX5_GET(tracer_string_event
, trace
, timestamp
);
489 tracer_event
->string_event
.string_param
=
490 MLX5_GET(tracer_string_event
, trace
, string_param
);
491 tracer_event
->string_event
.tmsn
=
492 MLX5_GET(tracer_string_event
, trace
, tmsn
);
493 tracer_event
->string_event
.tdsn
=
494 MLX5_GET(tracer_string_event
, trace
, tdsn
);
496 tracer_event
->type
= TRACER_EVENT_TYPE_UNRECOGNIZED
;
502 static u64
get_block_timestamp(struct mlx5_fw_tracer
*tracer
, u64
*ts_event
)
504 struct tracer_event tracer_event
;
507 event_id
= MLX5_GET(tracer_event
, ts_event
, event_id
);
509 if (event_id
== TRACER_EVENT_TYPE_TIMESTAMP
)
510 poll_trace(tracer
, &tracer_event
, ts_event
);
512 tracer_event
.timestamp_event
.timestamp
= 0;
514 return tracer_event
.timestamp_event
.timestamp
;
517 static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer
*tracer
)
519 struct tracer_string_format
*str_frmt
;
520 struct hlist_node
*n
;
523 for (i
= 0; i
< MESSAGE_HASH_SIZE
; i
++) {
524 hlist_for_each_entry_safe(str_frmt
, n
, &tracer
->hash
[i
], hlist
)
525 mlx5_tracer_clean_message(str_frmt
);
529 static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer
*tracer
)
531 struct tracer_string_format
*str_frmt
, *tmp_str
;
533 list_for_each_entry_safe(str_frmt
, tmp_str
, &tracer
->ready_strings_list
,
535 list_del(&str_frmt
->list
);
538 static void mlx5_fw_tracer_save_trace(struct mlx5_fw_tracer
*tracer
,
539 u64 timestamp
, bool lost
,
540 u8 event_id
, char *msg
)
542 struct mlx5_fw_trace_data
*trace_data
;
544 mutex_lock(&tracer
->st_arr
.lock
);
545 trace_data
= &tracer
->st_arr
.straces
[tracer
->st_arr
.saved_traces_index
];
546 trace_data
->timestamp
= timestamp
;
547 trace_data
->lost
= lost
;
548 trace_data
->event_id
= event_id
;
549 strscpy_pad(trace_data
->msg
, msg
, TRACE_STR_MSG
);
551 tracer
->st_arr
.saved_traces_index
=
552 (tracer
->st_arr
.saved_traces_index
+ 1) & (SAVED_TRACES_NUM
- 1);
553 mutex_unlock(&tracer
->st_arr
.lock
);
557 void mlx5_tracer_print_trace(struct tracer_string_format
*str_frmt
,
558 struct mlx5_core_dev
*dev
,
563 snprintf(tmp
, sizeof(tmp
), str_frmt
->string
,
570 str_frmt
->params
[6]);
572 trace_mlx5_fw(dev
->tracer
, trace_timestamp
, str_frmt
->lost
,
573 str_frmt
->event_id
, tmp
);
575 mlx5_fw_tracer_save_trace(dev
->tracer
, trace_timestamp
,
576 str_frmt
->lost
, str_frmt
->event_id
, tmp
);
578 /* remove it from hash */
579 mlx5_tracer_clean_message(str_frmt
);
582 static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer
*tracer
,
583 struct tracer_event
*tracer_event
)
585 struct tracer_string_format
*cur_string
;
587 if (tracer_event
->string_event
.tdsn
== 0) {
588 cur_string
= mlx5_tracer_get_string(tracer
, tracer_event
);
592 cur_string
->num_of_params
= mlx5_tracer_get_num_of_params(cur_string
->string
);
593 cur_string
->last_param_num
= 0;
594 cur_string
->event_id
= tracer_event
->event_id
;
595 cur_string
->tmsn
= tracer_event
->string_event
.tmsn
;
596 cur_string
->timestamp
= tracer_event
->string_event
.timestamp
;
597 cur_string
->lost
= tracer_event
->lost_event
;
598 if (cur_string
->num_of_params
== 0) /* trace with no params */
599 list_add_tail(&cur_string
->list
, &tracer
->ready_strings_list
);
601 cur_string
= mlx5_tracer_message_get(tracer
, tracer_event
);
603 pr_debug("%s Got string event for unknown string tdsm: %d\n",
604 __func__
, tracer_event
->string_event
.tmsn
);
607 cur_string
->last_param_num
+= 1;
608 if (cur_string
->last_param_num
> TRACER_MAX_PARAMS
) {
609 pr_debug("%s Number of params exceeds the max (%d)\n",
610 __func__
, TRACER_MAX_PARAMS
);
611 list_add_tail(&cur_string
->list
, &tracer
->ready_strings_list
);
614 /* keep the new parameter */
615 cur_string
->params
[cur_string
->last_param_num
- 1] =
616 tracer_event
->string_event
.string_param
;
617 if (cur_string
->last_param_num
== cur_string
->num_of_params
)
618 list_add_tail(&cur_string
->list
, &tracer
->ready_strings_list
);
624 static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer
*tracer
,
625 struct tracer_event
*tracer_event
)
627 struct tracer_timestamp_event timestamp_event
=
628 tracer_event
->timestamp_event
;
629 struct tracer_string_format
*str_frmt
, *tmp_str
;
630 struct mlx5_core_dev
*dev
= tracer
->dev
;
633 list_for_each_entry_safe(str_frmt
, tmp_str
, &tracer
->ready_strings_list
, list
) {
634 list_del(&str_frmt
->list
);
635 if (str_frmt
->timestamp
< (timestamp_event
.timestamp
& MASK_6_0
))
636 trace_timestamp
= (timestamp_event
.timestamp
& MASK_52_7
) |
637 (str_frmt
->timestamp
& MASK_6_0
);
639 trace_timestamp
= ((timestamp_event
.timestamp
& MASK_52_7
) - 1) |
640 (str_frmt
->timestamp
& MASK_6_0
);
642 mlx5_tracer_print_trace(str_frmt
, dev
, trace_timestamp
);
646 static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer
*tracer
,
647 struct tracer_event
*tracer_event
)
649 if (tracer_event
->type
== TRACER_EVENT_TYPE_STRING
) {
650 mlx5_tracer_handle_string_trace(tracer
, tracer_event
);
651 } else if (tracer_event
->type
== TRACER_EVENT_TYPE_TIMESTAMP
) {
652 if (!tracer_event
->timestamp_event
.unreliable
)
653 mlx5_tracer_handle_timestamp_trace(tracer
, tracer_event
);
655 pr_debug("%s Got unrecognised type %d for parsing, exiting..\n",
656 __func__
, tracer_event
->type
);
661 static void mlx5_fw_tracer_handle_traces(struct work_struct
*work
)
663 struct mlx5_fw_tracer
*tracer
=
664 container_of(work
, struct mlx5_fw_tracer
, handle_traces_work
);
665 u64 block_timestamp
, last_block_timestamp
, tmp_trace_block
[TRACES_PER_BLOCK
];
666 u32 block_count
, start_offset
, prev_start_offset
, prev_consumer_index
;
667 u32 trace_event_size
= MLX5_ST_SZ_BYTES(tracer_event
);
668 struct mlx5_core_dev
*dev
= tracer
->dev
;
669 struct tracer_event tracer_event
;
672 mlx5_core_dbg(dev
, "FWTracer: Handle Trace event, owner=(%d)\n", tracer
->owner
);
676 block_count
= tracer
->buff
.size
/ TRACER_BLOCK_SIZE_BYTE
;
677 start_offset
= tracer
->buff
.consumer_index
* TRACER_BLOCK_SIZE_BYTE
;
679 /* Copy the block to local buffer to avoid HW override while being processed*/
680 memcpy(tmp_trace_block
, tracer
->buff
.log_buf
+ start_offset
,
681 TRACER_BLOCK_SIZE_BYTE
);
684 get_block_timestamp(tracer
, &tmp_trace_block
[TRACES_PER_BLOCK
- 1]);
686 while (block_timestamp
> tracer
->last_timestamp
) {
687 /* Check block override if its not the first block */
688 if (!tracer
->last_timestamp
) {
690 /* To avoid block override be the HW in case of buffer
691 * wraparound, the time stamp of the previous block
692 * should be compared to the last timestamp handled
695 prev_consumer_index
=
696 (tracer
->buff
.consumer_index
- 1) & (block_count
- 1);
697 prev_start_offset
= prev_consumer_index
* TRACER_BLOCK_SIZE_BYTE
;
699 ts_event
= tracer
->buff
.log_buf
+ prev_start_offset
+
700 (TRACES_PER_BLOCK
- 1) * trace_event_size
;
701 last_block_timestamp
= get_block_timestamp(tracer
, ts_event
);
702 /* If previous timestamp different from last stored
703 * timestamp then there is a good chance that the
704 * current buffer is overwritten and therefore should
707 if (tracer
->last_timestamp
!= last_block_timestamp
) {
708 mlx5_core_warn(dev
, "FWTracer: Events were lost\n");
709 tracer
->last_timestamp
= block_timestamp
;
710 tracer
->buff
.consumer_index
=
711 (tracer
->buff
.consumer_index
+ 1) & (block_count
- 1);
717 for (i
= 0; i
< TRACES_PER_BLOCK
; i
++) {
718 poll_trace(tracer
, &tracer_event
, &tmp_trace_block
[i
]);
719 mlx5_tracer_handle_trace(tracer
, &tracer_event
);
722 tracer
->buff
.consumer_index
=
723 (tracer
->buff
.consumer_index
+ 1) & (block_count
- 1);
725 tracer
->last_timestamp
= block_timestamp
;
726 start_offset
= tracer
->buff
.consumer_index
* TRACER_BLOCK_SIZE_BYTE
;
727 memcpy(tmp_trace_block
, tracer
->buff
.log_buf
+ start_offset
,
728 TRACER_BLOCK_SIZE_BYTE
);
729 block_timestamp
= get_block_timestamp(tracer
,
730 &tmp_trace_block
[TRACES_PER_BLOCK
- 1]);
733 mlx5_fw_tracer_arm(dev
);
736 static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer
*tracer
)
738 struct mlx5_core_dev
*dev
= tracer
->dev
;
739 u32 out
[MLX5_ST_SZ_DW(mtrc_conf
)] = {0};
740 u32 in
[MLX5_ST_SZ_DW(mtrc_conf
)] = {0};
743 MLX5_SET(mtrc_conf
, in
, trace_mode
, TRACE_TO_MEMORY
);
744 MLX5_SET(mtrc_conf
, in
, log_trace_buffer_size
,
745 ilog2(TRACER_BUFFER_PAGE_NUM
));
746 MLX5_SET(mtrc_conf
, in
, trace_mkey
, tracer
->buff
.mkey
.key
);
748 err
= mlx5_core_access_reg(dev
, in
, sizeof(in
), out
, sizeof(out
),
749 MLX5_REG_MTRC_CONF
, 0, 1);
751 mlx5_core_warn(dev
, "FWTracer: Failed to set tracer configurations %d\n", err
);
756 static int mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer
*tracer
, u8 status
, u8 arm
)
758 struct mlx5_core_dev
*dev
= tracer
->dev
;
759 u32 out
[MLX5_ST_SZ_DW(mtrc_ctrl
)] = {0};
760 u32 in
[MLX5_ST_SZ_DW(mtrc_ctrl
)] = {0};
763 MLX5_SET(mtrc_ctrl
, in
, modify_field_select
, TRACE_STATUS
);
764 MLX5_SET(mtrc_ctrl
, in
, trace_status
, status
);
765 MLX5_SET(mtrc_ctrl
, in
, arm_event
, arm
);
767 err
= mlx5_core_access_reg(dev
, in
, sizeof(in
), out
, sizeof(out
),
768 MLX5_REG_MTRC_CTRL
, 0, 1);
771 tracer
->last_timestamp
= 0;
776 static int mlx5_fw_tracer_start(struct mlx5_fw_tracer
*tracer
)
778 struct mlx5_core_dev
*dev
= tracer
->dev
;
781 err
= mlx5_fw_tracer_ownership_acquire(tracer
);
783 mlx5_core_dbg(dev
, "FWTracer: Ownership was not granted %d\n", err
);
784 /* Don't fail since ownership can be acquired on a later FW event */
788 err
= mlx5_fw_tracer_set_mtrc_conf(tracer
);
790 mlx5_core_warn(dev
, "FWTracer: Failed to set tracer configuration %d\n", err
);
791 goto release_ownership
;
794 /* enable tracer & trace events */
795 err
= mlx5_fw_tracer_set_mtrc_ctrl(tracer
, 1, 1);
797 mlx5_core_warn(dev
, "FWTracer: Failed to enable tracer %d\n", err
);
798 goto release_ownership
;
801 mlx5_core_dbg(dev
, "FWTracer: Ownership granted and active\n");
805 mlx5_fw_tracer_ownership_release(tracer
);
809 static void mlx5_fw_tracer_ownership_change(struct work_struct
*work
)
811 struct mlx5_fw_tracer
*tracer
=
812 container_of(work
, struct mlx5_fw_tracer
, ownership_change_work
);
814 mlx5_core_dbg(tracer
->dev
, "FWTracer: ownership changed, current=(%d)\n", tracer
->owner
);
816 tracer
->owner
= false;
817 tracer
->buff
.consumer_index
= 0;
821 mlx5_fw_tracer_start(tracer
);
824 static int mlx5_fw_tracer_set_core_dump_reg(struct mlx5_core_dev
*dev
,
825 u32
*in
, int size_in
)
827 u32 out
[MLX5_ST_SZ_DW(core_dump_reg
)] = {};
829 if (!MLX5_CAP_DEBUG(dev
, core_dump_general
) &&
830 !MLX5_CAP_DEBUG(dev
, core_dump_qp
))
833 return mlx5_core_access_reg(dev
, in
, size_in
, out
, sizeof(out
),
834 MLX5_REG_CORE_DUMP
, 0, 1);
837 int mlx5_fw_tracer_trigger_core_dump_general(struct mlx5_core_dev
*dev
)
839 struct mlx5_fw_tracer
*tracer
= dev
->tracer
;
840 u32 in
[MLX5_ST_SZ_DW(core_dump_reg
)] = {};
843 if (!MLX5_CAP_DEBUG(dev
, core_dump_general
) || !tracer
)
848 MLX5_SET(core_dump_reg
, in
, core_dump_type
, 0x0);
850 err
= mlx5_fw_tracer_set_core_dump_reg(dev
, in
, sizeof(in
));
853 queue_work(tracer
->work_queue
, &tracer
->handle_traces_work
);
854 flush_workqueue(tracer
->work_queue
);
859 mlx5_devlink_fmsg_fill_trace(struct devlink_fmsg
*fmsg
,
860 struct mlx5_fw_trace_data
*trace_data
)
864 err
= devlink_fmsg_obj_nest_start(fmsg
);
868 err
= devlink_fmsg_u64_pair_put(fmsg
, "timestamp", trace_data
->timestamp
);
872 err
= devlink_fmsg_bool_pair_put(fmsg
, "lost", trace_data
->lost
);
876 err
= devlink_fmsg_u8_pair_put(fmsg
, "event_id", trace_data
->event_id
);
880 err
= devlink_fmsg_string_pair_put(fmsg
, "msg", trace_data
->msg
);
884 err
= devlink_fmsg_obj_nest_end(fmsg
);
890 int mlx5_fw_tracer_get_saved_traces_objects(struct mlx5_fw_tracer
*tracer
,
891 struct devlink_fmsg
*fmsg
)
893 struct mlx5_fw_trace_data
*straces
= tracer
->st_arr
.straces
;
894 u32 index
, start_index
, end_index
;
895 u32 saved_traces_index
;
898 if (!straces
[0].timestamp
)
901 mutex_lock(&tracer
->st_arr
.lock
);
902 saved_traces_index
= tracer
->st_arr
.saved_traces_index
;
903 if (straces
[saved_traces_index
].timestamp
)
904 start_index
= saved_traces_index
;
907 end_index
= (saved_traces_index
- 1) & (SAVED_TRACES_NUM
- 1);
909 err
= devlink_fmsg_arr_pair_nest_start(fmsg
, "dump fw traces");
913 while (index
!= end_index
) {
914 err
= mlx5_devlink_fmsg_fill_trace(fmsg
, &straces
[index
]);
918 index
= (index
+ 1) & (SAVED_TRACES_NUM
- 1);
921 err
= devlink_fmsg_arr_pair_nest_end(fmsg
);
923 mutex_unlock(&tracer
->st_arr
.lock
);
927 /* Create software resources (Buffers, etc ..) */
928 struct mlx5_fw_tracer
*mlx5_fw_tracer_create(struct mlx5_core_dev
*dev
)
930 struct mlx5_fw_tracer
*tracer
= NULL
;
933 if (!MLX5_CAP_MCAM_REG(dev
, tracer_registers
)) {
934 mlx5_core_dbg(dev
, "FWTracer: Tracer capability not present\n");
938 tracer
= kvzalloc(sizeof(*tracer
), GFP_KERNEL
);
940 return ERR_PTR(-ENOMEM
);
942 tracer
->work_queue
= create_singlethread_workqueue("mlx5_fw_tracer");
943 if (!tracer
->work_queue
) {
950 INIT_LIST_HEAD(&tracer
->ready_strings_list
);
951 INIT_WORK(&tracer
->ownership_change_work
, mlx5_fw_tracer_ownership_change
);
952 INIT_WORK(&tracer
->read_fw_strings_work
, mlx5_tracer_read_strings_db
);
953 INIT_WORK(&tracer
->handle_traces_work
, mlx5_fw_tracer_handle_traces
);
956 err
= mlx5_query_mtrc_caps(tracer
);
958 mlx5_core_dbg(dev
, "FWTracer: Failed to query capabilities %d\n", err
);
959 goto destroy_workqueue
;
962 err
= mlx5_fw_tracer_create_log_buf(tracer
);
964 mlx5_core_warn(dev
, "FWTracer: Create log buffer failed %d\n", err
);
965 goto destroy_workqueue
;
968 err
= mlx5_fw_tracer_allocate_strings_db(tracer
);
970 mlx5_core_warn(dev
, "FWTracer: Allocate strings database failed %d\n", err
);
974 mlx5_fw_tracer_init_saved_traces_array(tracer
);
975 mlx5_core_dbg(dev
, "FWTracer: Tracer created\n");
980 mlx5_fw_tracer_destroy_log_buf(tracer
);
983 destroy_workqueue(tracer
->work_queue
);
989 static int fw_tracer_event(struct notifier_block
*nb
, unsigned long action
, void *data
);
991 /* Create HW resources + start tracer */
992 int mlx5_fw_tracer_init(struct mlx5_fw_tracer
*tracer
)
994 struct mlx5_core_dev
*dev
;
997 if (IS_ERR_OR_NULL(tracer
))
1002 if (!tracer
->str_db
.loaded
)
1003 queue_work(tracer
->work_queue
, &tracer
->read_fw_strings_work
);
1005 err
= mlx5_core_alloc_pd(dev
, &tracer
->buff
.pdn
);
1007 mlx5_core_warn(dev
, "FWTracer: Failed to allocate PD %d\n", err
);
1011 err
= mlx5_fw_tracer_create_mkey(tracer
);
1013 mlx5_core_warn(dev
, "FWTracer: Failed to create mkey %d\n", err
);
1014 goto err_dealloc_pd
;
1017 MLX5_NB_INIT(&tracer
->nb
, fw_tracer_event
, DEVICE_TRACER
);
1018 mlx5_eq_notifier_register(dev
, &tracer
->nb
);
1020 mlx5_fw_tracer_start(tracer
);
1025 mlx5_core_dealloc_pd(dev
, tracer
->buff
.pdn
);
1029 /* Stop tracer + Cleanup HW resources */
1030 void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer
*tracer
)
1032 if (IS_ERR_OR_NULL(tracer
))
1035 mlx5_core_dbg(tracer
->dev
, "FWTracer: Cleanup, is owner ? (%d)\n",
1037 mlx5_eq_notifier_unregister(tracer
->dev
, &tracer
->nb
);
1038 cancel_work_sync(&tracer
->ownership_change_work
);
1039 cancel_work_sync(&tracer
->handle_traces_work
);
1042 mlx5_fw_tracer_ownership_release(tracer
);
1044 mlx5_core_destroy_mkey(tracer
->dev
, &tracer
->buff
.mkey
);
1045 mlx5_core_dealloc_pd(tracer
->dev
, tracer
->buff
.pdn
);
1048 /* Free software resources (Buffers, etc ..) */
1049 void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer
*tracer
)
1051 if (IS_ERR_OR_NULL(tracer
))
1054 mlx5_core_dbg(tracer
->dev
, "FWTracer: Destroy\n");
1056 cancel_work_sync(&tracer
->read_fw_strings_work
);
1057 mlx5_fw_tracer_clean_ready_list(tracer
);
1058 mlx5_fw_tracer_clean_print_hash(tracer
);
1059 mlx5_fw_tracer_clean_saved_traces_array(tracer
);
1060 mlx5_fw_tracer_free_strings_db(tracer
);
1061 mlx5_fw_tracer_destroy_log_buf(tracer
);
1062 flush_workqueue(tracer
->work_queue
);
1063 destroy_workqueue(tracer
->work_queue
);
1067 static int fw_tracer_event(struct notifier_block
*nb
, unsigned long action
, void *data
)
1069 struct mlx5_fw_tracer
*tracer
= mlx5_nb_cof(nb
, struct mlx5_fw_tracer
, nb
);
1070 struct mlx5_core_dev
*dev
= tracer
->dev
;
1071 struct mlx5_eqe
*eqe
= data
;
1073 switch (eqe
->sub_type
) {
1074 case MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE
:
1075 if (test_bit(MLX5_INTERFACE_STATE_UP
, &dev
->intf_state
))
1076 queue_work(tracer
->work_queue
, &tracer
->ownership_change_work
);
1078 case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE
:
1079 if (likely(tracer
->str_db
.loaded
))
1080 queue_work(tracer
->work_queue
, &tracer
->handle_traces_work
);
1083 mlx5_core_dbg(dev
, "FWTracer: Event with unrecognized subtype: sub_type %d\n",
1090 EXPORT_TRACEPOINT_SYMBOL(mlx5_fw
);