1 // SPDX-License-Identifier: GPL-2.0
3 * Driver for HiSilicon PCIe tune and trace device
5 * Copyright (c) 2022 HiSilicon Technologies Co., Ltd.
6 * Author: Yicong Yang <yangyicong@hisilicon.com>
9 #include <linux/bitfield.h>
10 #include <linux/bitops.h>
11 #include <linux/cpuhotplug.h>
12 #include <linux/delay.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/interrupt.h>
16 #include <linux/iommu.h>
17 #include <linux/iopoll.h>
18 #include <linux/module.h>
19 #include <linux/sysfs.h>
20 #include <linux/vmalloc.h>
24 /* Dynamic CPU hotplug state used by PTT */
25 static enum cpuhp_state hisi_ptt_pmu_online
;
27 static bool hisi_ptt_wait_tuning_finish(struct hisi_ptt
*hisi_ptt
)
31 return !readl_poll_timeout(hisi_ptt
->iobase
+ HISI_PTT_TUNING_INT_STAT
,
32 val
, !(val
& HISI_PTT_TUNING_INT_STAT_MASK
),
33 HISI_PTT_WAIT_POLL_INTERVAL_US
,
34 HISI_PTT_WAIT_TUNE_TIMEOUT_US
);
37 static ssize_t
hisi_ptt_tune_attr_show(struct device
*dev
,
38 struct device_attribute
*attr
,
41 struct hisi_ptt
*hisi_ptt
= to_hisi_ptt(dev_get_drvdata(dev
));
42 struct dev_ext_attribute
*ext_attr
;
43 struct hisi_ptt_tune_desc
*desc
;
47 ext_attr
= container_of(attr
, struct dev_ext_attribute
, attr
);
50 mutex_lock(&hisi_ptt
->tune_lock
);
52 reg
= readl(hisi_ptt
->iobase
+ HISI_PTT_TUNING_CTRL
);
53 reg
&= ~(HISI_PTT_TUNING_CTRL_CODE
| HISI_PTT_TUNING_CTRL_SUB
);
54 reg
|= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE
| HISI_PTT_TUNING_CTRL_SUB
,
56 writel(reg
, hisi_ptt
->iobase
+ HISI_PTT_TUNING_CTRL
);
58 /* Write all 1 to indicates it's the read process */
59 writel(~0U, hisi_ptt
->iobase
+ HISI_PTT_TUNING_DATA
);
61 if (!hisi_ptt_wait_tuning_finish(hisi_ptt
)) {
62 mutex_unlock(&hisi_ptt
->tune_lock
);
66 reg
= readl(hisi_ptt
->iobase
+ HISI_PTT_TUNING_DATA
);
67 reg
&= HISI_PTT_TUNING_DATA_VAL_MASK
;
68 val
= FIELD_GET(HISI_PTT_TUNING_DATA_VAL_MASK
, reg
);
70 mutex_unlock(&hisi_ptt
->tune_lock
);
71 return sysfs_emit(buf
, "%u\n", val
);
74 static ssize_t
hisi_ptt_tune_attr_store(struct device
*dev
,
75 struct device_attribute
*attr
,
76 const char *buf
, size_t count
)
78 struct hisi_ptt
*hisi_ptt
= to_hisi_ptt(dev_get_drvdata(dev
));
79 struct dev_ext_attribute
*ext_attr
;
80 struct hisi_ptt_tune_desc
*desc
;
84 ext_attr
= container_of(attr
, struct dev_ext_attribute
, attr
);
87 if (kstrtou16(buf
, 10, &val
))
90 mutex_lock(&hisi_ptt
->tune_lock
);
92 reg
= readl(hisi_ptt
->iobase
+ HISI_PTT_TUNING_CTRL
);
93 reg
&= ~(HISI_PTT_TUNING_CTRL_CODE
| HISI_PTT_TUNING_CTRL_SUB
);
94 reg
|= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE
| HISI_PTT_TUNING_CTRL_SUB
,
96 writel(reg
, hisi_ptt
->iobase
+ HISI_PTT_TUNING_CTRL
);
97 writel(FIELD_PREP(HISI_PTT_TUNING_DATA_VAL_MASK
, val
),
98 hisi_ptt
->iobase
+ HISI_PTT_TUNING_DATA
);
100 if (!hisi_ptt_wait_tuning_finish(hisi_ptt
)) {
101 mutex_unlock(&hisi_ptt
->tune_lock
);
105 mutex_unlock(&hisi_ptt
->tune_lock
);
109 #define HISI_PTT_TUNE_ATTR(_name, _val, _show, _store) \
110 static struct hisi_ptt_tune_desc _name##_desc = { \
112 .event_code = (_val), \
114 static struct dev_ext_attribute hisi_ptt_##_name##_attr = { \
115 .attr = __ATTR(_name, 0600, _show, _store), \
116 .var = &_name##_desc, \
119 #define HISI_PTT_TUNE_ATTR_COMMON(_name, _val) \
120 HISI_PTT_TUNE_ATTR(_name, _val, \
121 hisi_ptt_tune_attr_show, \
122 hisi_ptt_tune_attr_store)
125 * The value of the tuning event are composed of two parts: main event code
126 * in BIT[0,15] and subevent code in BIT[16,23]. For example, qox_tx_cpl is
127 * a subevent of 'Tx path QoS control' which for tuning the weight of Tx
128 * completion TLPs. See hisi_ptt.rst documentation for more information.
130 #define HISI_PTT_TUNE_QOS_TX_CPL (0x4 | (3 << 16))
131 #define HISI_PTT_TUNE_QOS_TX_NP (0x4 | (4 << 16))
132 #define HISI_PTT_TUNE_QOS_TX_P (0x4 | (5 << 16))
133 #define HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL (0x5 | (6 << 16))
134 #define HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL (0x5 | (7 << 16))
136 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_cpl
, HISI_PTT_TUNE_QOS_TX_CPL
);
137 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_np
, HISI_PTT_TUNE_QOS_TX_NP
);
138 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_p
, HISI_PTT_TUNE_QOS_TX_P
);
139 HISI_PTT_TUNE_ATTR_COMMON(rx_alloc_buf_level
, HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL
);
140 HISI_PTT_TUNE_ATTR_COMMON(tx_alloc_buf_level
, HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL
);
142 static struct attribute
*hisi_ptt_tune_attrs
[] = {
143 &hisi_ptt_qos_tx_cpl_attr
.attr
.attr
,
144 &hisi_ptt_qos_tx_np_attr
.attr
.attr
,
145 &hisi_ptt_qos_tx_p_attr
.attr
.attr
,
146 &hisi_ptt_rx_alloc_buf_level_attr
.attr
.attr
,
147 &hisi_ptt_tx_alloc_buf_level_attr
.attr
.attr
,
151 static struct attribute_group hisi_ptt_tune_group
= {
153 .attrs
= hisi_ptt_tune_attrs
,
156 static u16
hisi_ptt_get_filter_val(u16 devid
, bool is_port
)
159 return BIT(HISI_PCIE_CORE_PORT_ID(devid
& 0xff));
164 static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt
*hisi_ptt
)
168 return !readl_poll_timeout_atomic(hisi_ptt
->iobase
+ HISI_PTT_TRACE_STS
,
169 val
, val
& HISI_PTT_TRACE_IDLE
,
170 HISI_PTT_WAIT_POLL_INTERVAL_US
,
171 HISI_PTT_WAIT_TRACE_TIMEOUT_US
);
174 static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt
*hisi_ptt
)
178 readl_poll_timeout_atomic(hisi_ptt
->iobase
+ HISI_PTT_TRACE_WR_STS
,
179 val
, !val
, HISI_PTT_RESET_POLL_INTERVAL_US
,
180 HISI_PTT_RESET_TIMEOUT_US
);
183 static void hisi_ptt_trace_end(struct hisi_ptt
*hisi_ptt
)
185 writel(0, hisi_ptt
->iobase
+ HISI_PTT_TRACE_CTRL
);
186 hisi_ptt
->trace_ctrl
.started
= false;
189 static int hisi_ptt_trace_start(struct hisi_ptt
*hisi_ptt
)
191 struct hisi_ptt_trace_ctrl
*ctrl
= &hisi_ptt
->trace_ctrl
;
195 /* Check device idle before start trace */
196 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt
)) {
197 pci_err(hisi_ptt
->pdev
, "Failed to start trace, the device is still busy\n");
201 ctrl
->started
= true;
203 /* Reset the DMA before start tracing */
204 val
= readl(hisi_ptt
->iobase
+ HISI_PTT_TRACE_CTRL
);
205 val
|= HISI_PTT_TRACE_CTRL_RST
;
206 writel(val
, hisi_ptt
->iobase
+ HISI_PTT_TRACE_CTRL
);
208 hisi_ptt_wait_dma_reset_done(hisi_ptt
);
210 val
= readl(hisi_ptt
->iobase
+ HISI_PTT_TRACE_CTRL
);
211 val
&= ~HISI_PTT_TRACE_CTRL_RST
;
212 writel(val
, hisi_ptt
->iobase
+ HISI_PTT_TRACE_CTRL
);
214 /* Reset the index of current buffer */
215 hisi_ptt
->trace_ctrl
.buf_index
= 0;
217 /* Zero the trace buffers */
218 for (i
= 0; i
< HISI_PTT_TRACE_BUF_CNT
; i
++)
219 memset(ctrl
->trace_buf
[i
].addr
, 0, HISI_PTT_TRACE_BUF_SIZE
);
221 /* Clear the interrupt status */
222 writel(HISI_PTT_TRACE_INT_STAT_MASK
, hisi_ptt
->iobase
+ HISI_PTT_TRACE_INT_STAT
);
223 writel(0, hisi_ptt
->iobase
+ HISI_PTT_TRACE_INT_MASK
);
225 /* Set the trace control register */
226 val
= FIELD_PREP(HISI_PTT_TRACE_CTRL_TYPE_SEL
, ctrl
->type
);
227 val
|= FIELD_PREP(HISI_PTT_TRACE_CTRL_RXTX_SEL
, ctrl
->direction
);
228 val
|= FIELD_PREP(HISI_PTT_TRACE_CTRL_DATA_FORMAT
, ctrl
->format
);
229 val
|= FIELD_PREP(HISI_PTT_TRACE_CTRL_TARGET_SEL
, hisi_ptt
->trace_ctrl
.filter
);
230 if (!hisi_ptt
->trace_ctrl
.is_port
)
231 val
|= HISI_PTT_TRACE_CTRL_FILTER_MODE
;
233 /* Start the Trace */
234 val
|= HISI_PTT_TRACE_CTRL_EN
;
235 writel(val
, hisi_ptt
->iobase
+ HISI_PTT_TRACE_CTRL
);
240 static int hisi_ptt_update_aux(struct hisi_ptt
*hisi_ptt
, int index
, bool stop
)
242 struct hisi_ptt_trace_ctrl
*ctrl
= &hisi_ptt
->trace_ctrl
;
243 struct perf_output_handle
*handle
= &ctrl
->handle
;
244 struct perf_event
*event
= handle
->event
;
245 struct hisi_ptt_pmu_buf
*buf
;
249 buf
= perf_get_aux(handle
);
250 if (!buf
|| !handle
->size
)
253 addr
= ctrl
->trace_buf
[ctrl
->buf_index
].addr
;
256 * If we're going to stop, read the size of already traced data from
257 * HISI_PTT_TRACE_WR_STS. Otherwise we're coming from the interrupt,
258 * the data size is always HISI_PTT_TRACE_BUF_SIZE.
263 reg
= readl(hisi_ptt
->iobase
+ HISI_PTT_TRACE_WR_STS
);
264 size
= FIELD_GET(HISI_PTT_TRACE_WR_STS_WRITE
, reg
);
266 size
= HISI_PTT_TRACE_BUF_SIZE
;
269 memcpy(buf
->base
+ buf
->pos
, addr
, size
);
273 * Just commit the traced data if we're going to stop. Otherwise if the
274 * resident AUX buffer cannot contain the data of next trace buffer,
278 perf_aux_output_end(handle
, buf
->pos
);
279 } else if (buf
->length
- buf
->pos
< HISI_PTT_TRACE_BUF_SIZE
) {
280 perf_aux_output_end(handle
, buf
->pos
);
282 buf
= perf_aux_output_begin(handle
, event
);
286 buf
->pos
= handle
->head
% buf
->length
;
287 if (buf
->length
- buf
->pos
< HISI_PTT_TRACE_BUF_SIZE
) {
288 perf_aux_output_end(handle
, 0);
296 static irqreturn_t
hisi_ptt_isr(int irq
, void *context
)
298 struct hisi_ptt
*hisi_ptt
= context
;
301 status
= readl(hisi_ptt
->iobase
+ HISI_PTT_TRACE_INT_STAT
);
302 if (!(status
& HISI_PTT_TRACE_INT_STAT_MASK
))
305 buf_idx
= ffs(status
) - 1;
307 /* Clear the interrupt status of buffer @buf_idx */
308 writel(status
, hisi_ptt
->iobase
+ HISI_PTT_TRACE_INT_STAT
);
311 * Update the AUX buffer and cache the current buffer index,
312 * as we need to know this and save the data when the trace
313 * is ended out of the interrupt handler. End the trace
314 * if the updating fails.
316 if (hisi_ptt_update_aux(hisi_ptt
, buf_idx
, false))
317 hisi_ptt_trace_end(hisi_ptt
);
319 hisi_ptt
->trace_ctrl
.buf_index
= (buf_idx
+ 1) % HISI_PTT_TRACE_BUF_CNT
;
324 static void hisi_ptt_irq_free_vectors(void *pdev
)
326 pci_free_irq_vectors(pdev
);
329 static int hisi_ptt_register_irq(struct hisi_ptt
*hisi_ptt
)
331 struct pci_dev
*pdev
= hisi_ptt
->pdev
;
334 ret
= pci_alloc_irq_vectors(pdev
, 1, 1, PCI_IRQ_MSI
);
336 pci_err(pdev
, "failed to allocate irq vector, ret = %d\n", ret
);
340 ret
= devm_add_action_or_reset(&pdev
->dev
, hisi_ptt_irq_free_vectors
, pdev
);
344 hisi_ptt
->trace_irq
= pci_irq_vector(pdev
, HISI_PTT_TRACE_DMA_IRQ
);
345 ret
= devm_request_threaded_irq(&pdev
->dev
, hisi_ptt
->trace_irq
,
346 NULL
, hisi_ptt_isr
, 0,
349 pci_err(pdev
, "failed to request irq %d, ret = %d\n",
350 hisi_ptt
->trace_irq
, ret
);
357 static void hisi_ptt_del_free_filter(struct hisi_ptt
*hisi_ptt
,
358 struct hisi_ptt_filter_desc
*filter
)
361 hisi_ptt
->port_mask
&= ~hisi_ptt_get_filter_val(filter
->devid
, true);
363 list_del(&filter
->list
);
368 static struct hisi_ptt_filter_desc
*
369 hisi_ptt_alloc_add_filter(struct hisi_ptt
*hisi_ptt
, u16 devid
, bool is_port
)
371 struct hisi_ptt_filter_desc
*filter
;
372 u8 devfn
= devid
& 0xff;
375 filter_name
= kasprintf(GFP_KERNEL
, "%04x:%02x:%02x.%d", pci_domain_nr(hisi_ptt
->pdev
->bus
),
376 PCI_BUS_NUM(devid
), PCI_SLOT(devfn
), PCI_FUNC(devfn
));
378 pci_err(hisi_ptt
->pdev
, "failed to allocate name for filter %04x:%02x:%02x.%d\n",
379 pci_domain_nr(hisi_ptt
->pdev
->bus
), PCI_BUS_NUM(devid
),
380 PCI_SLOT(devfn
), PCI_FUNC(devfn
));
384 filter
= kzalloc(sizeof(*filter
), GFP_KERNEL
);
386 pci_err(hisi_ptt
->pdev
, "failed to add filter for %s\n",
392 filter
->name
= filter_name
;
393 filter
->is_port
= is_port
;
394 filter
->devid
= devid
;
396 if (filter
->is_port
) {
397 list_add_tail(&filter
->list
, &hisi_ptt
->port_filters
);
399 /* Update the available port mask */
400 hisi_ptt
->port_mask
|= hisi_ptt_get_filter_val(filter
->devid
, true);
402 list_add_tail(&filter
->list
, &hisi_ptt
->req_filters
);
408 static ssize_t
hisi_ptt_filter_show(struct device
*dev
, struct device_attribute
*attr
,
411 struct hisi_ptt_filter_desc
*filter
;
412 unsigned long filter_val
;
414 filter
= container_of(attr
, struct hisi_ptt_filter_desc
, attr
);
415 filter_val
= hisi_ptt_get_filter_val(filter
->devid
, filter
->is_port
) |
416 (filter
->is_port
? HISI_PTT_PMU_FILTER_IS_PORT
: 0);
418 return sysfs_emit(buf
, "0x%05lx\n", filter_val
);
421 static int hisi_ptt_create_rp_filter_attr(struct hisi_ptt
*hisi_ptt
,
422 struct hisi_ptt_filter_desc
*filter
)
424 struct kobject
*kobj
= &hisi_ptt
->hisi_ptt_pmu
.dev
->kobj
;
426 sysfs_attr_init(&filter
->attr
.attr
);
427 filter
->attr
.attr
.name
= filter
->name
;
428 filter
->attr
.attr
.mode
= 0400; /* DEVICE_ATTR_ADMIN_RO */
429 filter
->attr
.show
= hisi_ptt_filter_show
;
431 return sysfs_add_file_to_group(kobj
, &filter
->attr
.attr
,
432 HISI_PTT_RP_FILTERS_GRP_NAME
);
435 static void hisi_ptt_remove_rp_filter_attr(struct hisi_ptt
*hisi_ptt
,
436 struct hisi_ptt_filter_desc
*filter
)
438 struct kobject
*kobj
= &hisi_ptt
->hisi_ptt_pmu
.dev
->kobj
;
440 sysfs_remove_file_from_group(kobj
, &filter
->attr
.attr
,
441 HISI_PTT_RP_FILTERS_GRP_NAME
);
444 static int hisi_ptt_create_req_filter_attr(struct hisi_ptt
*hisi_ptt
,
445 struct hisi_ptt_filter_desc
*filter
)
447 struct kobject
*kobj
= &hisi_ptt
->hisi_ptt_pmu
.dev
->kobj
;
449 sysfs_attr_init(&filter
->attr
.attr
);
450 filter
->attr
.attr
.name
= filter
->name
;
451 filter
->attr
.attr
.mode
= 0400; /* DEVICE_ATTR_ADMIN_RO */
452 filter
->attr
.show
= hisi_ptt_filter_show
;
454 return sysfs_add_file_to_group(kobj
, &filter
->attr
.attr
,
455 HISI_PTT_REQ_FILTERS_GRP_NAME
);
458 static void hisi_ptt_remove_req_filter_attr(struct hisi_ptt
*hisi_ptt
,
459 struct hisi_ptt_filter_desc
*filter
)
461 struct kobject
*kobj
= &hisi_ptt
->hisi_ptt_pmu
.dev
->kobj
;
463 sysfs_remove_file_from_group(kobj
, &filter
->attr
.attr
,
464 HISI_PTT_REQ_FILTERS_GRP_NAME
);
467 static int hisi_ptt_create_filter_attr(struct hisi_ptt
*hisi_ptt
,
468 struct hisi_ptt_filter_desc
*filter
)
473 ret
= hisi_ptt_create_rp_filter_attr(hisi_ptt
, filter
);
475 ret
= hisi_ptt_create_req_filter_attr(hisi_ptt
, filter
);
478 pci_err(hisi_ptt
->pdev
, "failed to create sysfs attribute for filter %s\n",
484 static void hisi_ptt_remove_filter_attr(struct hisi_ptt
*hisi_ptt
,
485 struct hisi_ptt_filter_desc
*filter
)
488 hisi_ptt_remove_rp_filter_attr(hisi_ptt
, filter
);
490 hisi_ptt_remove_req_filter_attr(hisi_ptt
, filter
);
493 static void hisi_ptt_remove_all_filter_attributes(void *data
)
495 struct hisi_ptt_filter_desc
*filter
;
496 struct hisi_ptt
*hisi_ptt
= data
;
498 mutex_lock(&hisi_ptt
->filter_lock
);
500 list_for_each_entry(filter
, &hisi_ptt
->req_filters
, list
)
501 hisi_ptt_remove_filter_attr(hisi_ptt
, filter
);
503 list_for_each_entry(filter
, &hisi_ptt
->port_filters
, list
)
504 hisi_ptt_remove_filter_attr(hisi_ptt
, filter
);
506 hisi_ptt
->sysfs_inited
= false;
507 mutex_unlock(&hisi_ptt
->filter_lock
);
510 static int hisi_ptt_init_filter_attributes(struct hisi_ptt
*hisi_ptt
)
512 struct hisi_ptt_filter_desc
*filter
;
515 mutex_lock(&hisi_ptt
->filter_lock
);
518 * Register the reset callback in the first stage. In reset we traverse
519 * the filters list to remove the sysfs attributes so the callback can
520 * be called safely even without below filter attributes creation.
522 ret
= devm_add_action(&hisi_ptt
->pdev
->dev
,
523 hisi_ptt_remove_all_filter_attributes
,
528 list_for_each_entry(filter
, &hisi_ptt
->port_filters
, list
) {
529 ret
= hisi_ptt_create_filter_attr(hisi_ptt
, filter
);
534 list_for_each_entry(filter
, &hisi_ptt
->req_filters
, list
) {
535 ret
= hisi_ptt_create_filter_attr(hisi_ptt
, filter
);
540 hisi_ptt
->sysfs_inited
= true;
542 mutex_unlock(&hisi_ptt
->filter_lock
);
546 static void hisi_ptt_update_filters(struct work_struct
*work
)
548 struct delayed_work
*delayed_work
= to_delayed_work(work
);
549 struct hisi_ptt_filter_update_info info
;
550 struct hisi_ptt_filter_desc
*filter
;
551 struct hisi_ptt
*hisi_ptt
;
553 hisi_ptt
= container_of(delayed_work
, struct hisi_ptt
, work
);
555 if (!mutex_trylock(&hisi_ptt
->filter_lock
)) {
556 schedule_delayed_work(&hisi_ptt
->work
, HISI_PTT_WORK_DELAY_MS
);
560 while (kfifo_get(&hisi_ptt
->filter_update_kfifo
, &info
)) {
563 * Notify the users if failed to add this filter, others
564 * still work and available. See the comments in
565 * hisi_ptt_init_filters().
567 filter
= hisi_ptt_alloc_add_filter(hisi_ptt
, info
.devid
, info
.is_port
);
572 * If filters' sysfs entries hasn't been initialized,
573 * then we're still at probe stage. Add the filters to
574 * the list and later hisi_ptt_init_filter_attributes()
575 * will create sysfs attributes for all the filters.
577 if (hisi_ptt
->sysfs_inited
&&
578 hisi_ptt_create_filter_attr(hisi_ptt
, filter
)) {
579 hisi_ptt_del_free_filter(hisi_ptt
, filter
);
583 struct hisi_ptt_filter_desc
*tmp
;
584 struct list_head
*target_list
;
586 target_list
= info
.is_port
? &hisi_ptt
->port_filters
:
587 &hisi_ptt
->req_filters
;
589 list_for_each_entry_safe(filter
, tmp
, target_list
, list
)
590 if (filter
->devid
== info
.devid
) {
591 if (hisi_ptt
->sysfs_inited
)
592 hisi_ptt_remove_filter_attr(hisi_ptt
, filter
);
594 hisi_ptt_del_free_filter(hisi_ptt
, filter
);
600 mutex_unlock(&hisi_ptt
->filter_lock
);
604 * A PCI bus notifier is used here for dynamically updating the filter
607 static int hisi_ptt_notifier_call(struct notifier_block
*nb
, unsigned long action
,
610 struct hisi_ptt
*hisi_ptt
= container_of(nb
, struct hisi_ptt
, hisi_ptt_nb
);
611 struct hisi_ptt_filter_update_info info
;
612 struct pci_dev
*pdev
, *root_port
;
613 struct device
*dev
= data
;
616 pdev
= to_pci_dev(dev
);
617 root_port
= pcie_find_root_port(pdev
);
621 port_devid
= pci_dev_id(root_port
);
622 if (port_devid
< hisi_ptt
->lower_bdf
||
623 port_devid
> hisi_ptt
->upper_bdf
)
626 info
.is_port
= pci_pcie_type(pdev
) == PCI_EXP_TYPE_ROOT_PORT
;
627 info
.devid
= pci_dev_id(pdev
);
630 case BUS_NOTIFY_ADD_DEVICE
:
633 case BUS_NOTIFY_DEL_DEVICE
:
641 * The FIFO size is 16 which is sufficient for almost all the cases,
642 * since each PCIe core will have most 8 Root Ports (typically only
643 * 1~4 Root Ports). On failure log the failed filter and let user
646 if (kfifo_in_spinlocked(&hisi_ptt
->filter_update_kfifo
, &info
, 1,
647 &hisi_ptt
->filter_update_lock
))
648 schedule_delayed_work(&hisi_ptt
->work
, 0);
650 pci_warn(hisi_ptt
->pdev
,
651 "filter update fifo overflow for target %s\n",
657 static int hisi_ptt_init_filters(struct pci_dev
*pdev
, void *data
)
659 struct pci_dev
*root_port
= pcie_find_root_port(pdev
);
660 struct hisi_ptt_filter_desc
*filter
;
661 struct hisi_ptt
*hisi_ptt
= data
;
667 port_devid
= pci_dev_id(root_port
);
668 if (port_devid
< hisi_ptt
->lower_bdf
||
669 port_devid
> hisi_ptt
->upper_bdf
)
673 * We won't fail the probe if filter allocation failed here. The filters
674 * should be partial initialized and users would know which filter fails
675 * through the log. Other functions of PTT device are still available.
677 filter
= hisi_ptt_alloc_add_filter(hisi_ptt
, pci_dev_id(pdev
),
678 pci_pcie_type(pdev
) == PCI_EXP_TYPE_ROOT_PORT
);
685 static void hisi_ptt_release_filters(void *data
)
687 struct hisi_ptt_filter_desc
*filter
, *tmp
;
688 struct hisi_ptt
*hisi_ptt
= data
;
690 list_for_each_entry_safe(filter
, tmp
, &hisi_ptt
->req_filters
, list
)
691 hisi_ptt_del_free_filter(hisi_ptt
, filter
);
693 list_for_each_entry_safe(filter
, tmp
, &hisi_ptt
->port_filters
, list
)
694 hisi_ptt_del_free_filter(hisi_ptt
, filter
);
697 static int hisi_ptt_config_trace_buf(struct hisi_ptt
*hisi_ptt
)
699 struct hisi_ptt_trace_ctrl
*ctrl
= &hisi_ptt
->trace_ctrl
;
700 struct device
*dev
= &hisi_ptt
->pdev
->dev
;
703 ctrl
->trace_buf
= devm_kcalloc(dev
, HISI_PTT_TRACE_BUF_CNT
,
704 sizeof(*ctrl
->trace_buf
), GFP_KERNEL
);
705 if (!ctrl
->trace_buf
)
708 for (i
= 0; i
< HISI_PTT_TRACE_BUF_CNT
; ++i
) {
709 ctrl
->trace_buf
[i
].addr
= dmam_alloc_coherent(dev
, HISI_PTT_TRACE_BUF_SIZE
,
710 &ctrl
->trace_buf
[i
].dma
,
712 if (!ctrl
->trace_buf
[i
].addr
)
716 /* Configure the trace DMA buffer */
717 for (i
= 0; i
< HISI_PTT_TRACE_BUF_CNT
; i
++) {
718 writel(lower_32_bits(ctrl
->trace_buf
[i
].dma
),
719 hisi_ptt
->iobase
+ HISI_PTT_TRACE_ADDR_BASE_LO_0
+
720 i
* HISI_PTT_TRACE_ADDR_STRIDE
);
721 writel(upper_32_bits(ctrl
->trace_buf
[i
].dma
),
722 hisi_ptt
->iobase
+ HISI_PTT_TRACE_ADDR_BASE_HI_0
+
723 i
* HISI_PTT_TRACE_ADDR_STRIDE
);
725 writel(HISI_PTT_TRACE_BUF_SIZE
, hisi_ptt
->iobase
+ HISI_PTT_TRACE_ADDR_SIZE
);
730 static int hisi_ptt_init_ctrls(struct hisi_ptt
*hisi_ptt
)
732 struct pci_dev
*pdev
= hisi_ptt
->pdev
;
737 INIT_DELAYED_WORK(&hisi_ptt
->work
, hisi_ptt_update_filters
);
738 INIT_KFIFO(hisi_ptt
->filter_update_kfifo
);
739 spin_lock_init(&hisi_ptt
->filter_update_lock
);
741 INIT_LIST_HEAD(&hisi_ptt
->port_filters
);
742 INIT_LIST_HEAD(&hisi_ptt
->req_filters
);
743 mutex_init(&hisi_ptt
->filter_lock
);
745 ret
= hisi_ptt_config_trace_buf(hisi_ptt
);
750 * The device range register provides the information about the root
751 * ports which the RCiEP can control and trace. The RCiEP and the root
752 * ports which it supports are on the same PCIe core, with same domain
753 * number but maybe different bus number. The device range register
754 * will tell us which root ports we can support, Bit[31:16] indicates
755 * the upper BDF numbers of the root port, while Bit[15:0] indicates
758 reg
= readl(hisi_ptt
->iobase
+ HISI_PTT_DEVICE_RANGE
);
759 hisi_ptt
->upper_bdf
= FIELD_GET(HISI_PTT_DEVICE_RANGE_UPPER
, reg
);
760 hisi_ptt
->lower_bdf
= FIELD_GET(HISI_PTT_DEVICE_RANGE_LOWER
, reg
);
762 bus
= pci_find_bus(pci_domain_nr(pdev
->bus
), PCI_BUS_NUM(hisi_ptt
->upper_bdf
));
764 pci_walk_bus(bus
, hisi_ptt_init_filters
, hisi_ptt
);
766 ret
= devm_add_action_or_reset(&pdev
->dev
, hisi_ptt_release_filters
, hisi_ptt
);
770 hisi_ptt
->trace_ctrl
.on_cpu
= -1;
774 static ssize_t
cpumask_show(struct device
*dev
, struct device_attribute
*attr
,
777 struct hisi_ptt
*hisi_ptt
= to_hisi_ptt(dev_get_drvdata(dev
));
778 const cpumask_t
*cpumask
= cpumask_of_node(dev_to_node(&hisi_ptt
->pdev
->dev
));
780 return cpumap_print_to_pagebuf(true, buf
, cpumask
);
782 static DEVICE_ATTR_RO(cpumask
);
784 static struct attribute
*hisi_ptt_cpumask_attrs
[] = {
785 &dev_attr_cpumask
.attr
,
789 static const struct attribute_group hisi_ptt_cpumask_attr_group
= {
790 .attrs
= hisi_ptt_cpumask_attrs
,
794 * Bit 19 indicates the filter type, 1 for Root Port filter and 0 for Requester
795 * filter. Bit[15:0] indicates the filter value, for Root Port filter it's
796 * a bit mask of desired ports and for Requester filter it's the Requester ID
797 * of the desired PCIe function. Bit[18:16] is reserved for extension.
799 * See hisi_ptt.rst documentation for detailed information.
801 PMU_FORMAT_ATTR(filter
, "config:0-19");
802 PMU_FORMAT_ATTR(direction
, "config:20-23");
803 PMU_FORMAT_ATTR(type
, "config:24-31");
804 PMU_FORMAT_ATTR(format
, "config:32-35");
806 static struct attribute
*hisi_ptt_pmu_format_attrs
[] = {
807 &format_attr_filter
.attr
,
808 &format_attr_direction
.attr
,
809 &format_attr_type
.attr
,
810 &format_attr_format
.attr
,
814 static struct attribute_group hisi_ptt_pmu_format_group
= {
816 .attrs
= hisi_ptt_pmu_format_attrs
,
819 static ssize_t
hisi_ptt_filter_multiselect_show(struct device
*dev
,
820 struct device_attribute
*attr
,
823 struct dev_ext_attribute
*ext_attr
;
825 ext_attr
= container_of(attr
, struct dev_ext_attribute
, attr
);
826 return sysfs_emit(buf
, "%s\n", (char *)ext_attr
->var
);
829 static struct dev_ext_attribute root_port_filters_multiselect
= {
831 .attr
= { .name
= "multiselect", .mode
= 0400 },
832 .show
= hisi_ptt_filter_multiselect_show
,
837 static struct attribute
*hisi_ptt_pmu_root_ports_attrs
[] = {
838 &root_port_filters_multiselect
.attr
.attr
,
842 static struct attribute_group hisi_ptt_pmu_root_ports_group
= {
843 .name
= HISI_PTT_RP_FILTERS_GRP_NAME
,
844 .attrs
= hisi_ptt_pmu_root_ports_attrs
,
847 static struct dev_ext_attribute requester_filters_multiselect
= {
849 .attr
= { .name
= "multiselect", .mode
= 0400 },
850 .show
= hisi_ptt_filter_multiselect_show
,
855 static struct attribute
*hisi_ptt_pmu_requesters_attrs
[] = {
856 &requester_filters_multiselect
.attr
.attr
,
860 static struct attribute_group hisi_ptt_pmu_requesters_group
= {
861 .name
= HISI_PTT_REQ_FILTERS_GRP_NAME
,
862 .attrs
= hisi_ptt_pmu_requesters_attrs
,
865 static const struct attribute_group
*hisi_ptt_pmu_groups
[] = {
866 &hisi_ptt_cpumask_attr_group
,
867 &hisi_ptt_pmu_format_group
,
868 &hisi_ptt_tune_group
,
869 &hisi_ptt_pmu_root_ports_group
,
870 &hisi_ptt_pmu_requesters_group
,
874 static int hisi_ptt_trace_valid_direction(u32 val
)
877 * The direction values have different effects according to the data
878 * format (specified in the parentheses). TLP set A/B means different
879 * set of TLP types. See hisi_ptt.rst documentation for more details.
881 static const u32 hisi_ptt_trace_available_direction
[] = {
882 0, /* inbound(4DW) or reserved(8DW) */
883 1, /* outbound(4DW) */
884 2, /* {in, out}bound(4DW) or inbound(8DW), TLP set A */
885 3, /* {in, out}bound(4DW) or inbound(8DW), TLP set B */
889 for (i
= 0; i
< ARRAY_SIZE(hisi_ptt_trace_available_direction
); i
++) {
890 if (val
== hisi_ptt_trace_available_direction
[i
])
897 static int hisi_ptt_trace_valid_type(u32 val
)
899 /* Different types can be set simultaneously */
900 static const u32 hisi_ptt_trace_available_type
[] = {
901 1, /* posted_request */
902 2, /* non-posted_request */
911 * Walk the available list and clear the valid bits of
912 * the config. If there is any resident bit after the
913 * walk then the config is invalid.
915 for (i
= 0; i
< ARRAY_SIZE(hisi_ptt_trace_available_type
); i
++)
916 val
&= ~hisi_ptt_trace_available_type
[i
];
924 static int hisi_ptt_trace_valid_format(u32 val
)
926 static const u32 hisi_ptt_trace_availble_format
[] = {
932 for (i
= 0; i
< ARRAY_SIZE(hisi_ptt_trace_availble_format
); i
++) {
933 if (val
== hisi_ptt_trace_availble_format
[i
])
940 static int hisi_ptt_trace_valid_filter(struct hisi_ptt
*hisi_ptt
, u64 config
)
942 unsigned long val
, port_mask
= hisi_ptt
->port_mask
;
943 struct hisi_ptt_filter_desc
*filter
;
946 hisi_ptt
->trace_ctrl
.is_port
= FIELD_GET(HISI_PTT_PMU_FILTER_IS_PORT
, config
);
947 val
= FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK
, config
);
950 * Port filters are defined as bit mask. For port filters, check
951 * the bits in the @val are within the range of hisi_ptt->port_mask
952 * and whether it's empty or not, otherwise user has specified
953 * some unsupported root ports.
955 * For Requester ID filters, walk the available filter list to see
956 * whether we have one matched.
958 mutex_lock(&hisi_ptt
->filter_lock
);
959 if (!hisi_ptt
->trace_ctrl
.is_port
) {
960 list_for_each_entry(filter
, &hisi_ptt
->req_filters
, list
) {
961 if (val
== hisi_ptt_get_filter_val(filter
->devid
, filter
->is_port
))
964 } else if (bitmap_subset(&val
, &port_mask
, BITS_PER_LONG
)) {
970 mutex_unlock(&hisi_ptt
->filter_lock
);
974 static void hisi_ptt_pmu_init_configs(struct hisi_ptt
*hisi_ptt
, struct perf_event
*event
)
976 struct hisi_ptt_trace_ctrl
*ctrl
= &hisi_ptt
->trace_ctrl
;
979 val
= FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK
, event
->attr
.config
);
980 hisi_ptt
->trace_ctrl
.filter
= val
;
982 val
= FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK
, event
->attr
.config
);
983 ctrl
->direction
= val
;
985 val
= FIELD_GET(HISI_PTT_PMU_TYPE_MASK
, event
->attr
.config
);
988 val
= FIELD_GET(HISI_PTT_PMU_FORMAT_MASK
, event
->attr
.config
);
992 static int hisi_ptt_pmu_event_init(struct perf_event
*event
)
994 struct hisi_ptt
*hisi_ptt
= to_hisi_ptt(event
->pmu
);
998 if (event
->cpu
< 0) {
999 dev_dbg(event
->pmu
->dev
, "Per-task mode not supported\n");
1003 if (event
->attr
.type
!= hisi_ptt
->hisi_ptt_pmu
.type
)
1006 ret
= hisi_ptt_trace_valid_filter(hisi_ptt
, event
->attr
.config
);
1010 val
= FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK
, event
->attr
.config
);
1011 ret
= hisi_ptt_trace_valid_direction(val
);
1015 val
= FIELD_GET(HISI_PTT_PMU_TYPE_MASK
, event
->attr
.config
);
1016 ret
= hisi_ptt_trace_valid_type(val
);
1020 val
= FIELD_GET(HISI_PTT_PMU_FORMAT_MASK
, event
->attr
.config
);
1021 return hisi_ptt_trace_valid_format(val
);
1024 static void *hisi_ptt_pmu_setup_aux(struct perf_event
*event
, void **pages
,
1025 int nr_pages
, bool overwrite
)
1027 struct hisi_ptt_pmu_buf
*buf
;
1028 struct page
**pagelist
;
1032 dev_warn(event
->pmu
->dev
, "Overwrite mode is not supported\n");
1036 /* If the pages size less than buffers, we cannot start trace */
1037 if (nr_pages
< HISI_PTT_TRACE_TOTAL_BUF_SIZE
/ PAGE_SIZE
)
1040 buf
= kzalloc(sizeof(*buf
), GFP_KERNEL
);
1044 pagelist
= kcalloc(nr_pages
, sizeof(*pagelist
), GFP_KERNEL
);
1048 for (i
= 0; i
< nr_pages
; i
++)
1049 pagelist
[i
] = virt_to_page(pages
[i
]);
1051 buf
->base
= vmap(pagelist
, nr_pages
, VM_MAP
, PAGE_KERNEL
);
1057 buf
->nr_pages
= nr_pages
;
1058 buf
->length
= nr_pages
* PAGE_SIZE
;
1068 static void hisi_ptt_pmu_free_aux(void *aux
)
1070 struct hisi_ptt_pmu_buf
*buf
= aux
;
1076 static void hisi_ptt_pmu_start(struct perf_event
*event
, int flags
)
1078 struct hisi_ptt
*hisi_ptt
= to_hisi_ptt(event
->pmu
);
1079 struct perf_output_handle
*handle
= &hisi_ptt
->trace_ctrl
.handle
;
1080 struct hw_perf_event
*hwc
= &event
->hw
;
1081 struct device
*dev
= event
->pmu
->dev
;
1082 struct hisi_ptt_pmu_buf
*buf
;
1083 int cpu
= event
->cpu
;
1088 /* Serialize the perf process if user specified several CPUs */
1089 spin_lock(&hisi_ptt
->pmu_lock
);
1090 if (hisi_ptt
->trace_ctrl
.started
) {
1091 dev_dbg(dev
, "trace has already started\n");
1096 * Handle the interrupt on the same cpu which starts the trace to avoid
1097 * context mismatch. Otherwise we'll trigger the WARN from the perf
1098 * core in event_function_local(). If CPU passed is offline we'll fail
1099 * here, just log it since we can do nothing here.
1101 ret
= irq_set_affinity(hisi_ptt
->trace_irq
, cpumask_of(cpu
));
1103 dev_warn(dev
, "failed to set the affinity of trace interrupt\n");
1105 hisi_ptt
->trace_ctrl
.on_cpu
= cpu
;
1107 buf
= perf_aux_output_begin(handle
, event
);
1109 dev_dbg(dev
, "aux output begin failed\n");
1113 buf
->pos
= handle
->head
% buf
->length
;
1115 hisi_ptt_pmu_init_configs(hisi_ptt
, event
);
1117 ret
= hisi_ptt_trace_start(hisi_ptt
);
1119 dev_dbg(dev
, "trace start failed, ret = %d\n", ret
);
1120 perf_aux_output_end(handle
, 0);
1124 spin_unlock(&hisi_ptt
->pmu_lock
);
1127 event
->hw
.state
|= PERF_HES_STOPPED
;
1128 spin_unlock(&hisi_ptt
->pmu_lock
);
1131 static void hisi_ptt_pmu_stop(struct perf_event
*event
, int flags
)
1133 struct hisi_ptt
*hisi_ptt
= to_hisi_ptt(event
->pmu
);
1134 struct hw_perf_event
*hwc
= &event
->hw
;
1136 if (hwc
->state
& PERF_HES_STOPPED
)
1139 spin_lock(&hisi_ptt
->pmu_lock
);
1140 if (hisi_ptt
->trace_ctrl
.started
) {
1141 hisi_ptt_trace_end(hisi_ptt
);
1143 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt
))
1144 dev_warn(event
->pmu
->dev
, "Device is still busy\n");
1146 hisi_ptt_update_aux(hisi_ptt
, hisi_ptt
->trace_ctrl
.buf_index
, true);
1148 spin_unlock(&hisi_ptt
->pmu_lock
);
1150 hwc
->state
|= PERF_HES_STOPPED
;
1151 perf_event_update_userpage(event
);
1152 hwc
->state
|= PERF_HES_UPTODATE
;
1155 static int hisi_ptt_pmu_add(struct perf_event
*event
, int flags
)
1157 struct hisi_ptt
*hisi_ptt
= to_hisi_ptt(event
->pmu
);
1158 struct hw_perf_event
*hwc
= &event
->hw
;
1159 int cpu
= event
->cpu
;
1161 /* Only allow the cpus on the device's node to add the event */
1162 if (!cpumask_test_cpu(cpu
, cpumask_of_node(dev_to_node(&hisi_ptt
->pdev
->dev
))))
1165 hwc
->state
= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
1167 if (flags
& PERF_EF_START
) {
1168 hisi_ptt_pmu_start(event
, PERF_EF_RELOAD
);
1169 if (hwc
->state
& PERF_HES_STOPPED
)
1176 static void hisi_ptt_pmu_del(struct perf_event
*event
, int flags
)
1178 hisi_ptt_pmu_stop(event
, PERF_EF_UPDATE
);
1181 static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node
)
1183 cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online
, hotplug_node
);
1186 static void hisi_ptt_unregister_pmu(void *pmu
)
1188 perf_pmu_unregister(pmu
);
1191 static int hisi_ptt_register_pmu(struct hisi_ptt
*hisi_ptt
)
1193 u16 core_id
, sicl_id
;
1198 ret
= cpuhp_state_add_instance_nocalls(hisi_ptt_pmu_online
,
1199 &hisi_ptt
->hotplug_node
);
1203 ret
= devm_add_action_or_reset(&hisi_ptt
->pdev
->dev
,
1204 hisi_ptt_remove_cpuhp_instance
,
1205 &hisi_ptt
->hotplug_node
);
1209 mutex_init(&hisi_ptt
->tune_lock
);
1210 spin_lock_init(&hisi_ptt
->pmu_lock
);
1212 hisi_ptt
->hisi_ptt_pmu
= (struct pmu
) {
1213 .module
= THIS_MODULE
,
1214 .capabilities
= PERF_PMU_CAP_EXCLUSIVE
| PERF_PMU_CAP_NO_EXCLUDE
,
1215 .task_ctx_nr
= perf_sw_context
,
1216 .attr_groups
= hisi_ptt_pmu_groups
,
1217 .event_init
= hisi_ptt_pmu_event_init
,
1218 .setup_aux
= hisi_ptt_pmu_setup_aux
,
1219 .free_aux
= hisi_ptt_pmu_free_aux
,
1220 .start
= hisi_ptt_pmu_start
,
1221 .stop
= hisi_ptt_pmu_stop
,
1222 .add
= hisi_ptt_pmu_add
,
1223 .del
= hisi_ptt_pmu_del
,
1226 reg
= readl(hisi_ptt
->iobase
+ HISI_PTT_LOCATION
);
1227 core_id
= FIELD_GET(HISI_PTT_CORE_ID
, reg
);
1228 sicl_id
= FIELD_GET(HISI_PTT_SICL_ID
, reg
);
1230 pmu_name
= devm_kasprintf(&hisi_ptt
->pdev
->dev
, GFP_KERNEL
, "hisi_ptt%u_%u",
1235 ret
= perf_pmu_register(&hisi_ptt
->hisi_ptt_pmu
, pmu_name
, -1);
1239 return devm_add_action_or_reset(&hisi_ptt
->pdev
->dev
,
1240 hisi_ptt_unregister_pmu
,
1241 &hisi_ptt
->hisi_ptt_pmu
);
1244 static void hisi_ptt_unregister_filter_update_notifier(void *data
)
1246 struct hisi_ptt
*hisi_ptt
= data
;
1248 bus_unregister_notifier(&pci_bus_type
, &hisi_ptt
->hisi_ptt_nb
);
1250 /* Cancel any work that has been queued */
1251 cancel_delayed_work_sync(&hisi_ptt
->work
);
1254 /* Register the bus notifier for dynamically updating the filter list */
1255 static int hisi_ptt_register_filter_update_notifier(struct hisi_ptt
*hisi_ptt
)
1259 hisi_ptt
->hisi_ptt_nb
.notifier_call
= hisi_ptt_notifier_call
;
1260 ret
= bus_register_notifier(&pci_bus_type
, &hisi_ptt
->hisi_ptt_nb
);
1264 return devm_add_action_or_reset(&hisi_ptt
->pdev
->dev
,
1265 hisi_ptt_unregister_filter_update_notifier
,
1270 * The DMA of PTT trace can only use direct mappings due to some
1271 * hardware restriction. Check whether there is no IOMMU or the
1272 * policy of the IOMMU domain is passthrough, otherwise the trace
1275 * The PTT device is supposed to behind an ARM SMMUv3, which
1276 * should have passthrough the device by a quirk.
1278 static int hisi_ptt_check_iommu_mapping(struct pci_dev
*pdev
)
1280 struct iommu_domain
*iommu_domain
;
1282 iommu_domain
= iommu_get_domain_for_dev(&pdev
->dev
);
1283 if (!iommu_domain
|| iommu_domain
->type
== IOMMU_DOMAIN_IDENTITY
)
1289 static int hisi_ptt_probe(struct pci_dev
*pdev
,
1290 const struct pci_device_id
*id
)
1292 struct hisi_ptt
*hisi_ptt
;
1295 ret
= hisi_ptt_check_iommu_mapping(pdev
);
1297 pci_err(pdev
, "requires direct DMA mappings\n");
1301 hisi_ptt
= devm_kzalloc(&pdev
->dev
, sizeof(*hisi_ptt
), GFP_KERNEL
);
1305 hisi_ptt
->pdev
= pdev
;
1306 pci_set_drvdata(pdev
, hisi_ptt
);
1308 ret
= pcim_enable_device(pdev
);
1310 pci_err(pdev
, "failed to enable device, ret = %d\n", ret
);
1314 ret
= pcim_iomap_regions(pdev
, BIT(2), DRV_NAME
);
1316 pci_err(pdev
, "failed to remap io memory, ret = %d\n", ret
);
1320 hisi_ptt
->iobase
= pcim_iomap_table(pdev
)[2];
1322 ret
= dma_set_coherent_mask(&pdev
->dev
, DMA_BIT_MASK(64));
1324 pci_err(pdev
, "failed to set 64 bit dma mask, ret = %d\n", ret
);
1328 pci_set_master(pdev
);
1330 ret
= hisi_ptt_register_irq(hisi_ptt
);
1334 ret
= hisi_ptt_init_ctrls(hisi_ptt
);
1336 pci_err(pdev
, "failed to init controls, ret = %d\n", ret
);
1340 ret
= hisi_ptt_register_filter_update_notifier(hisi_ptt
);
1342 pci_warn(pdev
, "failed to register filter update notifier, ret = %d", ret
);
1344 ret
= hisi_ptt_register_pmu(hisi_ptt
);
1346 pci_err(pdev
, "failed to register PMU device, ret = %d", ret
);
1350 ret
= hisi_ptt_init_filter_attributes(hisi_ptt
);
1352 pci_err(pdev
, "failed to init sysfs filter attributes, ret = %d", ret
);
1359 static const struct pci_device_id hisi_ptt_id_tbl
[] = {
1360 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI
, 0xa12e) },
1363 MODULE_DEVICE_TABLE(pci
, hisi_ptt_id_tbl
);
1365 static struct pci_driver hisi_ptt_driver
= {
1367 .id_table
= hisi_ptt_id_tbl
,
1368 .probe
= hisi_ptt_probe
,
1371 static int hisi_ptt_cpu_teardown(unsigned int cpu
, struct hlist_node
*node
)
1373 struct hisi_ptt
*hisi_ptt
;
1377 hisi_ptt
= hlist_entry_safe(node
, struct hisi_ptt
, hotplug_node
);
1378 src
= hisi_ptt
->trace_ctrl
.on_cpu
;
1379 dev
= hisi_ptt
->hisi_ptt_pmu
.dev
;
1381 if (!hisi_ptt
->trace_ctrl
.started
|| src
!= cpu
)
1384 target
= cpumask_any_but(cpumask_of_node(dev_to_node(&hisi_ptt
->pdev
->dev
)), cpu
);
1385 if (target
>= nr_cpu_ids
) {
1386 dev_err(dev
, "no available cpu for perf context migration\n");
1390 perf_pmu_migrate_context(&hisi_ptt
->hisi_ptt_pmu
, src
, target
);
1393 * Also make sure the interrupt bind to the migrated CPU as well. Warn
1394 * the user on failure here.
1396 if (irq_set_affinity(hisi_ptt
->trace_irq
, cpumask_of(target
)))
1397 dev_warn(dev
, "failed to set the affinity of trace interrupt\n");
1399 hisi_ptt
->trace_ctrl
.on_cpu
= target
;
1403 static int __init
hisi_ptt_init(void)
1407 ret
= cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN
, DRV_NAME
, NULL
,
1408 hisi_ptt_cpu_teardown
);
1411 hisi_ptt_pmu_online
= ret
;
1413 ret
= pci_register_driver(&hisi_ptt_driver
);
1415 cpuhp_remove_multi_state(hisi_ptt_pmu_online
);
1419 module_init(hisi_ptt_init
);
1421 static void __exit
hisi_ptt_exit(void)
1423 pci_unregister_driver(&hisi_ptt_driver
);
1424 cpuhp_remove_multi_state(hisi_ptt_pmu_online
);
1426 module_exit(hisi_ptt_exit
);
1428 MODULE_LICENSE("GPL");
1429 MODULE_AUTHOR("Yicong Yang <yangyicong@hisilicon.com>");
1430 MODULE_DESCRIPTION("Driver for HiSilicon PCIe tune and trace device");