]>
Commit | Line | Data |
---|---|---|
ad0dfdfd MP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (c) 2014, The Linux Foundation. All rights reserved. | |
2e1cdfe1 PP |
4 | */ |
5 | ||
134124ac | 6 | #include <linux/acpi.h> |
e7255092 | 7 | #include <linux/bitops.h> |
2e1cdfe1 PP |
8 | #include <linux/kernel.h> |
9 | #include <linux/moduleparam.h> | |
10 | #include <linux/init.h> | |
11 | #include <linux/types.h> | |
12 | #include <linux/device.h> | |
13 | #include <linux/io.h> | |
14 | #include <linux/err.h> | |
15 | #include <linux/fs.h> | |
16 | #include <linux/slab.h> | |
17 | #include <linux/delay.h> | |
18 | #include <linux/smp.h> | |
19 | #include <linux/sysfs.h> | |
20 | #include <linux/stat.h> | |
21 | #include <linux/clk.h> | |
22 | #include <linux/cpu.h> | |
f188b5e7 | 23 | #include <linux/cpu_pm.h> |
2e1cdfe1 | 24 | #include <linux/coresight.h> |
fc208abe | 25 | #include <linux/coresight-pmu.h> |
2e1cdfe1 PP |
26 | #include <linux/pm_wakeup.h> |
27 | #include <linux/amba/bus.h> | |
28 | #include <linux/seq_file.h> | |
29 | #include <linux/uaccess.h> | |
37fbbdbd | 30 | #include <linux/perf_event.h> |
5214b563 | 31 | #include <linux/platform_device.h> |
2e1cdfe1 | 32 | #include <linux/pm_runtime.h> |
f188b5e7 | 33 | #include <linux/property.h> |
73d779a0 | 34 | #include <linux/clk/clk-conf.h> |
e7255092 | 35 | |
8b481196 | 36 | #include <asm/barrier.h> |
2e1cdfe1 | 37 | #include <asm/sections.h> |
e7255092 | 38 | #include <asm/sysreg.h> |
c38a9ec2 | 39 | #include <asm/local.h> |
b860801e | 40 | #include <asm/virt.h> |
2e1cdfe1 PP |
41 | |
42 | #include "coresight-etm4x.h" | |
37fbbdbd | 43 | #include "coresight-etm-perf.h" |
810ac401 | 44 | #include "coresight-etm4x-cfg.h" |
937d3f58 | 45 | #include "coresight-self-hosted-trace.h" |
810ac401 | 46 | #include "coresight-syscfg.h" |
df487120 | 47 | #include "coresight-trace-id.h" |
2e1cdfe1 PP |
48 | |
49 | static int boot_enable; | |
08d2ddaa AM |
50 | module_param(boot_enable, int, 0444); |
51 | MODULE_PARM_DESC(boot_enable, "Enable tracing on boot"); | |
2e1cdfe1 | 52 | |
f188b5e7 AM |
53 | #define PARAM_PM_SAVE_FIRMWARE 0 /* save self-hosted state as per firmware */ |
54 | #define PARAM_PM_SAVE_NEVER 1 /* never save any state */ | |
55 | #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */ | |
56 | ||
57 | static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE; | |
58 | module_param(pm_save_enable, int, 0444); | |
59 | MODULE_PARM_DESC(pm_save_enable, | |
60 | "Save/restore state on power down: 1 = never, 2 = self-hosted"); | |
61 | ||
2e1cdfe1 | 62 | static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; |
2703d74c MP |
63 | static void etm4_set_default_config(struct etmv4_config *config); |
64 | static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, | |
65 | struct perf_event *event); | |
096dcfb9 | 66 | static u64 etm4_get_access_type(struct etmv4_config *config); |
2e1cdfe1 | 67 | |
58eb457b SAS |
68 | static enum cpuhp_state hp_online; |
69 | ||
fd6e7905 | 70 | struct etm4_init_arg { |
3c728e07 | 71 | struct device *dev; |
fd6e7905 SP |
72 | struct csdev_access *csa; |
73 | }; | |
74 | ||
3c728e07 TZ |
75 | static DEFINE_PER_CPU(struct etm4_init_arg *, delayed_probe); |
76 | static int etm4_probe_cpu(unsigned int cpu); | |
77 | ||
f6a18f35 SP |
78 | /* |
79 | * Check if TRCSSPCICRn(i) is implemented for a given instance. | |
80 | * | |
81 | * TRCSSPCICRn is implemented only if : | |
82 | * TRCSSPCICR<n> is present only if all of the following are true: | |
83 | * TRCIDR4.NUMSSCC > n. | |
84 | * TRCIDR4.NUMPC > 0b0000 . | |
85 | * TRCSSCSR<n>.PC == 0b1 | |
86 | */ | |
87 | static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) | |
88 | { | |
89 | return (n < drvdata->nr_ss_cmp) && | |
90 | drvdata->nr_pe && | |
91 | (drvdata->config.ss_status[n] & TRCSSCSRn_PC); | |
92 | } | |
93 | ||
03336d0f SP |
94 | u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit) |
95 | { | |
96 | u64 res = 0; | |
97 | ||
98 | switch (offset) { | |
99 | ETM4x_READ_SYSREG_CASES(res) | |
100 | default : | |
101 | pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n", | |
102 | offset); | |
103 | } | |
104 | ||
105 | if (!_relaxed) | |
018b741e | 106 | __io_ar(res); /* Imitate the !relaxed I/O helpers */ |
03336d0f SP |
107 | |
108 | return res; | |
109 | } | |
110 | ||
111 | void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) | |
112 | { | |
113 | if (!_relaxed) | |
018b741e | 114 | __io_bw(); /* Imitate the !relaxed I/O helpers */ |
03336d0f SP |
115 | if (!_64bit) |
116 | val &= GENMASK(31, 0); | |
117 | ||
118 | switch (offset) { | |
119 | ETM4x_WRITE_SYSREG_CASES(val) | |
120 | default : | |
121 | pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n", | |
122 | offset); | |
123 | } | |
124 | } | |
125 | ||
3e666ad0 | 126 | static u64 ete_sysreg_read(u32 offset, bool _relaxed, bool _64bit) |
2e1cdfe1 | 127 | { |
3e666ad0 SP |
128 | u64 res = 0; |
129 | ||
130 | switch (offset) { | |
131 | ETE_READ_CASES(res) | |
132 | default : | |
133 | pr_warn_ratelimited("ete: trying to read unsupported register @%x\n", | |
134 | offset); | |
135 | } | |
136 | ||
137 | if (!_relaxed) | |
018b741e | 138 | __io_ar(res); /* Imitate the !relaxed I/O helpers */ |
3e666ad0 SP |
139 | |
140 | return res; | |
141 | } | |
142 | ||
143 | static void ete_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) | |
144 | { | |
145 | if (!_relaxed) | |
018b741e | 146 | __io_bw(); /* Imitate the !relaxed I/O helpers */ |
3e666ad0 SP |
147 | if (!_64bit) |
148 | val &= GENMASK(31, 0); | |
149 | ||
150 | switch (offset) { | |
151 | ETE_WRITE_CASES(val) | |
152 | default : | |
153 | pr_warn_ratelimited("ete: trying to write to unsupported register @%x\n", | |
154 | offset); | |
155 | } | |
156 | } | |
157 | ||
bc2c689f SP |
158 | static void etm_detect_os_lock(struct etmv4_drvdata *drvdata, |
159 | struct csdev_access *csa) | |
2e1cdfe1 | 160 | { |
bc2c689f SP |
161 | u32 oslsr = etm4x_relaxed_read32(csa, TRCOSLSR); |
162 | ||
163 | drvdata->os_lock_model = ETM_OSLSR_OSLM(oslsr); | |
164 | } | |
165 | ||
166 | static void etm_write_os_lock(struct etmv4_drvdata *drvdata, | |
167 | struct csdev_access *csa, u32 val) | |
168 | { | |
169 | val = !!val; | |
170 | ||
171 | switch (drvdata->os_lock_model) { | |
172 | case ETM_OSLOCK_PRESENT: | |
173 | etm4x_relaxed_write32(csa, val, TRCOSLAR); | |
174 | break; | |
175 | case ETM_OSLOCK_PE: | |
176 | write_sysreg_s(val, SYS_OSLAR_EL1); | |
177 | break; | |
178 | default: | |
179 | pr_warn_once("CPU%d: Unsupported Trace OSLock model: %x\n", | |
180 | smp_processor_id(), drvdata->os_lock_model); | |
181 | fallthrough; | |
182 | case ETM_OSLOCK_NI: | |
183 | return; | |
184 | } | |
2e1cdfe1 PP |
185 | isb(); |
186 | } | |
187 | ||
bc2c689f SP |
188 | static inline void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata, |
189 | struct csdev_access *csa) | |
190 | { | |
191 | WARN_ON(drvdata->cpu != smp_processor_id()); | |
192 | ||
193 | /* Writing 0 to OS Lock unlocks the trace unit registers */ | |
194 | etm_write_os_lock(drvdata, csa, 0x0); | |
195 | drvdata->os_unlock = true; | |
196 | } | |
197 | ||
f5bd5236 SP |
198 | static void etm4_os_unlock(struct etmv4_drvdata *drvdata) |
199 | { | |
200 | if (!WARN_ON(!drvdata->csdev)) | |
201 | etm4_os_unlock_csa(drvdata, &drvdata->csdev->access); | |
f5bd5236 SP |
202 | } |
203 | ||
f188b5e7 AM |
204 | static void etm4_os_lock(struct etmv4_drvdata *drvdata) |
205 | { | |
f5bd5236 SP |
206 | if (WARN_ON(!drvdata->csdev)) |
207 | return; | |
bc2c689f SP |
208 | /* Writing 0x1 to OS Lock locks the trace registers */ |
209 | etm_write_os_lock(drvdata, &drvdata->csdev->access, 0x1); | |
f188b5e7 | 210 | drvdata->os_unlock = false; |
f188b5e7 AM |
211 | } |
212 | ||
33d5573a SP |
213 | static void etm4_cs_lock(struct etmv4_drvdata *drvdata, |
214 | struct csdev_access *csa) | |
215 | { | |
216 | /* Software Lock is only accessible via memory mapped interface */ | |
217 | if (csa->io_mem) | |
218 | CS_LOCK(csa->base); | |
219 | } | |
220 | ||
221 | static void etm4_cs_unlock(struct etmv4_drvdata *drvdata, | |
222 | struct csdev_access *csa) | |
223 | { | |
224 | if (csa->io_mem) | |
225 | CS_UNLOCK(csa->base); | |
226 | } | |
227 | ||
52210c87 MP |
228 | static int etm4_cpu_id(struct coresight_device *csdev) |
229 | { | |
230 | struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); | |
231 | ||
232 | return drvdata->cpu; | |
233 | } | |
234 | ||
df487120 ML |
235 | int etm4_read_alloc_trace_id(struct etmv4_drvdata *drvdata) |
236 | { | |
237 | int trace_id; | |
238 | ||
239 | /* | |
240 | * This will allocate a trace ID to the cpu, | |
241 | * or return the one currently allocated. | |
242 | * The trace id function has its own lock | |
243 | */ | |
244 | trace_id = coresight_trace_id_get_cpu_id(drvdata->cpu); | |
245 | if (IS_VALID_CS_TRACE_ID(trace_id)) | |
246 | drvdata->trcid = (u8)trace_id; | |
247 | else | |
248 | dev_err(&drvdata->csdev->dev, | |
249 | "Failed to allocate trace ID for %s on CPU%d\n", | |
250 | dev_name(&drvdata->csdev->dev), drvdata->cpu); | |
251 | return trace_id; | |
252 | } | |
253 | ||
254 | void etm4_release_trace_id(struct etmv4_drvdata *drvdata) | |
255 | { | |
256 | coresight_trace_id_put_cpu_id(drvdata->cpu); | |
257 | } | |
258 | ||
e006d89a SP |
259 | struct etm4_enable_arg { |
260 | struct etmv4_drvdata *drvdata; | |
261 | int rc; | |
262 | }; | |
263 | ||
5f6fd1aa SP |
264 | /* |
265 | * etm4x_prohibit_trace - Prohibit the CPU from tracing at all ELs. | |
266 | * When the CPU supports FEAT_TRF, we could move the ETM to a trace | |
267 | * prohibited state by filtering the Exception levels via TRFCR_EL1. | |
268 | */ | |
269 | static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata) | |
270 | { | |
271 | /* If the CPU doesn't support FEAT_TRF, nothing to do */ | |
272 | if (!drvdata->trfcr) | |
273 | return; | |
274 | cpu_prohibit_trace(); | |
275 | } | |
276 | ||
277 | /* | |
278 | * etm4x_allow_trace - Allow CPU tracing in the respective ELs, | |
279 | * as configured by the drvdata->config.mode for the current | |
280 | * session. Even though we have TRCVICTLR bits to filter the | |
281 | * trace in the ELs, it doesn't prevent the ETM from generating | |
282 | * a packet (e.g, TraceInfo) that might contain the addresses from | |
283 | * the excluded levels. Thus we use the additional controls provided | |
284 | * via the Trace Filtering controls (FEAT_TRF) to make sure no trace | |
285 | * is generated for the excluded ELs. | |
286 | */ | |
287 | static void etm4x_allow_trace(struct etmv4_drvdata *drvdata) | |
288 | { | |
289 | u64 trfcr = drvdata->trfcr; | |
290 | ||
291 | /* If the CPU doesn't support FEAT_TRF, nothing to do */ | |
292 | if (!trfcr) | |
293 | return; | |
294 | ||
295 | if (drvdata->config.mode & ETM_MODE_EXCL_KERN) | |
296 | trfcr &= ~TRFCR_ELx_ExTRE; | |
297 | if (drvdata->config.mode & ETM_MODE_EXCL_USER) | |
298 | trfcr &= ~TRFCR_ELx_E0TRE; | |
299 | ||
300 | write_trfcr(trfcr); | |
301 | } | |
302 | ||
e7255092 QL |
303 | #ifdef CONFIG_ETM4X_IMPDEF_FEATURE |
304 | ||
305 | #define HISI_HIP08_AMBA_ID 0x000b6d01 | |
306 | #define ETM4_AMBA_MASK 0xfffff | |
307 | #define HISI_HIP08_CORE_COMMIT_MASK 0x3000 | |
308 | #define HISI_HIP08_CORE_COMMIT_SHIFT 12 | |
309 | #define HISI_HIP08_CORE_COMMIT_FULL 0b00 | |
310 | #define HISI_HIP08_CORE_COMMIT_LVL_1 0b01 | |
311 | #define HISI_HIP08_CORE_COMMIT_REG sys_reg(3, 1, 15, 2, 5) | |
312 | ||
313 | struct etm4_arch_features { | |
314 | void (*arch_callback)(bool enable); | |
315 | }; | |
316 | ||
317 | static bool etm4_hisi_match_pid(unsigned int id) | |
318 | { | |
319 | return (id & ETM4_AMBA_MASK) == HISI_HIP08_AMBA_ID; | |
320 | } | |
321 | ||
322 | static void etm4_hisi_config_core_commit(bool enable) | |
323 | { | |
324 | u8 commit = enable ? HISI_HIP08_CORE_COMMIT_LVL_1 : | |
325 | HISI_HIP08_CORE_COMMIT_FULL; | |
326 | u64 val; | |
327 | ||
328 | /* | |
329 | * bit 12 and 13 of HISI_HIP08_CORE_COMMIT_REG are used together | |
330 | * to set core-commit, 2'b00 means cpu is at full speed, 2'b01, | |
331 | * 2'b10, 2'b11 mean reduce pipeline speed, and 2'b01 means level-1 | |
332 | * speed(minimun value). So bit 12 and 13 should be cleared together. | |
333 | */ | |
334 | val = read_sysreg_s(HISI_HIP08_CORE_COMMIT_REG); | |
335 | val &= ~HISI_HIP08_CORE_COMMIT_MASK; | |
336 | val |= commit << HISI_HIP08_CORE_COMMIT_SHIFT; | |
337 | write_sysreg_s(val, HISI_HIP08_CORE_COMMIT_REG); | |
338 | } | |
339 | ||
340 | static struct etm4_arch_features etm4_features[] = { | |
341 | [ETM4_IMPDEF_HISI_CORE_COMMIT] = { | |
342 | .arch_callback = etm4_hisi_config_core_commit, | |
343 | }, | |
344 | {}, | |
345 | }; | |
346 | ||
347 | static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) | |
348 | { | |
349 | struct etm4_arch_features *ftr; | |
350 | int bit; | |
351 | ||
352 | for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) { | |
353 | ftr = &etm4_features[bit]; | |
354 | ||
355 | if (ftr->arch_callback) | |
356 | ftr->arch_callback(true); | |
357 | } | |
358 | } | |
359 | ||
360 | static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata) | |
361 | { | |
362 | struct etm4_arch_features *ftr; | |
363 | int bit; | |
364 | ||
365 | for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) { | |
366 | ftr = &etm4_features[bit]; | |
367 | ||
368 | if (ftr->arch_callback) | |
369 | ftr->arch_callback(false); | |
370 | } | |
371 | } | |
372 | ||
373 | static void etm4_check_arch_features(struct etmv4_drvdata *drvdata, | |
5a1c7097 | 374 | struct csdev_access *csa) |
e7255092 | 375 | { |
5a1c7097 AK |
376 | /* |
377 | * TRCPIDR* registers are not required for ETMs with system | |
378 | * instructions. They must be identified by the MIDR+REVIDRs. | |
379 | * Skip the TRCPID checks for now. | |
380 | */ | |
381 | if (!csa->io_mem) | |
382 | return; | |
383 | ||
384 | if (etm4_hisi_match_pid(coresight_get_pid(csa))) | |
e7255092 QL |
385 | set_bit(ETM4_IMPDEF_HISI_CORE_COMMIT, drvdata->arch_features); |
386 | } | |
387 | #else | |
388 | static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) | |
389 | { | |
390 | } | |
391 | ||
392 | static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata) | |
393 | { | |
394 | } | |
395 | ||
396 | static void etm4_check_arch_features(struct etmv4_drvdata *drvdata, | |
5a1c7097 | 397 | struct csdev_access *csa) |
e7255092 QL |
398 | { |
399 | } | |
400 | #endif /* CONFIG_ETM4X_IMPDEF_FEATURE */ | |
401 | ||
e006d89a | 402 | static int etm4_enable_hw(struct etmv4_drvdata *drvdata) |
2e1cdfe1 | 403 | { |
68a14775 | 404 | int i, rc; |
54ff892b | 405 | struct etmv4_config *config = &drvdata->config; |
02005282 SP |
406 | struct coresight_device *csdev = drvdata->csdev; |
407 | struct device *etm_dev = &csdev->dev; | |
408 | struct csdev_access *csa = &csdev->access; | |
2e1cdfe1 | 409 | |
33d5573a SP |
410 | |
411 | etm4_cs_unlock(drvdata, csa); | |
e7255092 | 412 | etm4_enable_arch_specific(drvdata); |
2e1cdfe1 PP |
413 | |
414 | etm4_os_unlock(drvdata); | |
415 | ||
8ce00296 | 416 | rc = coresight_claim_device_unlocked(csdev); |
68a14775 SP |
417 | if (rc) |
418 | goto done; | |
419 | ||
2e1cdfe1 | 420 | /* Disable the trace unit before programming trace registers */ |
f5bd5236 | 421 | etm4x_relaxed_write32(csa, 0, TRCPRGCTLR); |
2e1cdfe1 | 422 | |
1ab3bb9d SP |
423 | /* |
424 | * If we use system instructions, we need to synchronize the | |
425 | * write to the TRCPRGCTLR, before accessing the TRCSTATR. | |
426 | * See ARM IHI0064F, section | |
427 | * "4.3.7 Synchronization of register updates" | |
428 | */ | |
429 | if (!csa->io_mem) | |
430 | isb(); | |
431 | ||
2e1cdfe1 | 432 | /* wait for TRCSTATR.IDLE to go up */ |
02005282 | 433 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) |
aaff7623 | 434 | dev_err(etm_dev, |
67337e8d | 435 | "timeout while waiting for Idle Trace Status\n"); |
6288b4ce | 436 | if (drvdata->nr_pe) |
f5bd5236 SP |
437 | etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR); |
438 | etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR); | |
2e1cdfe1 | 439 | /* nothing specific implemented */ |
f5bd5236 SP |
440 | etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR); |
441 | etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R); | |
442 | etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R); | |
f7289606 SP |
443 | if (drvdata->stallctl) |
444 | etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR); | |
f5bd5236 SP |
445 | etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR); |
446 | etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR); | |
447 | etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR); | |
448 | etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR); | |
449 | etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR); | |
450 | etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR); | |
451 | etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR); | |
452 | etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); | |
60c519c5 | 453 | if (drvdata->nr_pe_cmp) |
f5bd5236 | 454 | etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); |
2e1cdfe1 | 455 | for (i = 0; i < drvdata->nrseqstate - 1; i++) |
f5bd5236 | 456 | etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); |
589d9282 JH |
457 | if (drvdata->nrseqstate) { |
458 | etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); | |
459 | etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR); | |
460 | } | |
f5bd5236 | 461 | etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR); |
2e1cdfe1 | 462 | for (i = 0; i < drvdata->nr_cntr; i++) { |
f5bd5236 SP |
463 | etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i)); |
464 | etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i)); | |
465 | etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i)); | |
2e1cdfe1 | 466 | } |
497b5956 | 467 | |
8013f32a MP |
468 | /* |
469 | * Resource selector pair 0 is always implemented and reserved. As | |
470 | * such start at 2. | |
471 | */ | |
472 | for (i = 2; i < drvdata->nr_resource * 2; i++) | |
f5bd5236 | 473 | etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i)); |
2e1cdfe1 PP |
474 | |
475 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { | |
ebddaad0 ML |
476 | /* always clear status bit on restart if using single-shot */ |
477 | if (config->ss_ctrl[i] || config->ss_pe_cmp[i]) | |
0544f32b | 478 | config->ss_status[i] &= ~TRCSSCSRn_STATUS; |
f5bd5236 SP |
479 | etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i)); |
480 | etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i)); | |
f6a18f35 | 481 | if (etm4x_sspcicrn_present(drvdata, i)) |
f5bd5236 | 482 | etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i)); |
2e1cdfe1 | 483 | } |
bf84937e | 484 | for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { |
f5bd5236 SP |
485 | etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i)); |
486 | etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i)); | |
2e1cdfe1 PP |
487 | } |
488 | for (i = 0; i < drvdata->numcidc; i++) | |
f5bd5236 SP |
489 | etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i)); |
490 | etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0); | |
f2603b22 | 491 | if (drvdata->numcidc > 4) |
f5bd5236 | 492 | etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1); |
2e1cdfe1 PP |
493 | |
494 | for (i = 0; i < drvdata->numvmidc; i++) | |
f5bd5236 SP |
495 | etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i)); |
496 | etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0); | |
93dd6440 | 497 | if (drvdata->numvmidc > 4) |
f5bd5236 | 498 | etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1); |
2e1cdfe1 | 499 | |
02510a5a | 500 | if (!drvdata->skip_power_up) { |
f5bd5236 SP |
501 | u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR); |
502 | ||
02510a5a TZ |
503 | /* |
504 | * Request to keep the trace unit powered and also | |
505 | * emulation of powerdown | |
506 | */ | |
f5bd5236 | 507 | etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR); |
02510a5a | 508 | } |
46a3d5cd | 509 | |
35e1c916 SP |
510 | /* |
511 | * ETE mandates that the TRCRSR is written to before | |
512 | * enabling it. | |
513 | */ | |
514 | if (etm4x_is_ete(drvdata)) | |
515 | etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR); | |
516 | ||
5f6fd1aa | 517 | etm4x_allow_trace(drvdata); |
2e1cdfe1 | 518 | /* Enable the trace unit */ |
f5bd5236 | 519 | etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); |
2e1cdfe1 | 520 | |
1ab3bb9d SP |
521 | /* Synchronize the register updates for sysreg access */ |
522 | if (!csa->io_mem) | |
523 | isb(); | |
524 | ||
2e1cdfe1 | 525 | /* wait for TRCSTATR.IDLE to go back down to '0' */ |
02005282 | 526 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) |
aaff7623 | 527 | dev_err(etm_dev, |
67337e8d | 528 | "timeout while waiting for Idle Trace Status\n"); |
2e1cdfe1 | 529 | |
1004ce4c AM |
530 | /* |
531 | * As recommended by section 4.3.7 ("Synchronization when using the | |
532 | * memory-mapped interface") of ARM IHI 0064D | |
533 | */ | |
534 | dsb(sy); | |
535 | isb(); | |
536 | ||
68a14775 | 537 | done: |
33d5573a | 538 | etm4_cs_lock(drvdata, csa); |
2e1cdfe1 | 539 | |
aaff7623 | 540 | dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n", |
68a14775 SP |
541 | drvdata->cpu, rc); |
542 | return rc; | |
e006d89a SP |
543 | } |
544 | ||
545 | static void etm4_enable_hw_smp_call(void *info) | |
546 | { | |
547 | struct etm4_enable_arg *arg = info; | |
548 | ||
549 | if (WARN_ON(!arg)) | |
550 | return; | |
551 | arg->rc = etm4_enable_hw(arg->drvdata); | |
2e1cdfe1 PP |
552 | } |
553 | ||
a54e14f8 MP |
554 | /* |
555 | * The goal of function etm4_config_timestamp_event() is to configure a | |
556 | * counter that will tell the tracer to emit a timestamp packet when it | |
557 | * reaches zero. This is done in order to get a more fine grained idea | |
558 | * of when instructions are executed so that they can be correlated | |
559 | * with execution on other CPUs. | |
560 | * | |
561 | * To do this the counter itself is configured to self reload and | |
562 | * TRCRSCTLR1 (always true) used to get the counter to decrement. From | |
563 | * there a resource selector is configured with the counter and the | |
564 | * timestamp control register to use the resource selector to trigger the | |
565 | * event that will insert a timestamp packet in the stream. | |
566 | */ | |
567 | static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata) | |
568 | { | |
569 | int ctridx, ret = -EINVAL; | |
570 | int counter, rselector; | |
571 | u32 val = 0; | |
572 | struct etmv4_config *config = &drvdata->config; | |
573 | ||
574 | /* No point in trying if we don't have at least one counter */ | |
575 | if (!drvdata->nr_cntr) | |
576 | goto out; | |
577 | ||
578 | /* Find a counter that hasn't been initialised */ | |
579 | for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++) | |
580 | if (config->cntr_val[ctridx] == 0) | |
581 | break; | |
582 | ||
583 | /* All the counters have been configured already, bail out */ | |
584 | if (ctridx == drvdata->nr_cntr) { | |
585 | pr_debug("%s: no available counter found\n", __func__); | |
586 | ret = -ENOSPC; | |
587 | goto out; | |
588 | } | |
589 | ||
590 | /* | |
591 | * Searching for an available resource selector to use, starting at | |
592 | * '2' since every implementation has at least 2 resource selector. | |
593 | * ETMIDR4 gives the number of resource selector _pairs_, | |
594 | * hence multiply by 2. | |
595 | */ | |
596 | for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++) | |
597 | if (!config->res_ctrl[rselector]) | |
598 | break; | |
599 | ||
600 | if (rselector == drvdata->nr_resource * 2) { | |
601 | pr_debug("%s: no available resource selector found\n", | |
602 | __func__); | |
603 | ret = -ENOSPC; | |
604 | goto out; | |
605 | } | |
606 | ||
607 | /* Remember what counter we used */ | |
608 | counter = 1 << ctridx; | |
609 | ||
610 | /* | |
611 | * Initialise original and reload counter value to the smallest | |
612 | * possible value in order to get as much precision as we can. | |
613 | */ | |
614 | config->cntr_val[ctridx] = 1; | |
615 | config->cntrldvr[ctridx] = 1; | |
616 | ||
617 | /* Set the trace counter control register */ | |
618 | val = 0x1 << 16 | /* Bit 16, reload counter automatically */ | |
619 | 0x0 << 7 | /* Select single resource selector */ | |
620 | 0x1; /* Resource selector 1, i.e always true */ | |
621 | ||
622 | config->cntr_ctrl[ctridx] = val; | |
623 | ||
624 | val = 0x2 << 16 | /* Group 0b0010 - Counter and sequencers */ | |
625 | counter << 0; /* Counter to use */ | |
626 | ||
627 | config->res_ctrl[rselector] = val; | |
628 | ||
629 | val = 0x0 << 7 | /* Select single resource selector */ | |
630 | rselector; /* Resource selector */ | |
631 | ||
632 | config->ts_ctrl = val; | |
633 | ||
634 | ret = 0; | |
635 | out: | |
636 | return ret; | |
637 | } | |
638 | ||
810ac401 | 639 | static int etm4_parse_event_config(struct coresight_device *csdev, |
68905d73 | 640 | struct perf_event *event) |
37fbbdbd | 641 | { |
2703d74c | 642 | int ret = 0; |
810ac401 | 643 | struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); |
37fbbdbd | 644 | struct etmv4_config *config = &drvdata->config; |
68905d73 | 645 | struct perf_event_attr *attr = &event->attr; |
810ac401 ML |
646 | unsigned long cfg_hash; |
647 | int preset; | |
37fbbdbd | 648 | |
37fbbdbd MP |
649 | /* Clear configuration from previous run */ |
650 | memset(config, 0, sizeof(struct etmv4_config)); | |
651 | ||
652 | if (attr->exclude_kernel) | |
653 | config->mode = ETM_MODE_EXCL_KERN; | |
654 | ||
655 | if (attr->exclude_user) | |
656 | config->mode = ETM_MODE_EXCL_USER; | |
657 | ||
658 | /* Always start from the default config */ | |
2703d74c MP |
659 | etm4_set_default_config(config); |
660 | ||
661 | /* Configure filters specified on the perf cmd line, if any. */ | |
662 | ret = etm4_set_event_filters(drvdata, event); | |
663 | if (ret) | |
664 | goto out; | |
37fbbdbd | 665 | |
37fbbdbd | 666 | /* Go from generic option to ETMv4 specifics */ |
ae3fabcd | 667 | if (attr->config & BIT(ETM_OPT_CYCACC)) { |
1cf50f64 | 668 | config->cfg |= TRCCONFIGR_CCI; |
ae3fabcd ML |
669 | /* TRM: Must program this for cycacc to work */ |
670 | config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT; | |
671 | } | |
a54e14f8 MP |
672 | if (attr->config & BIT(ETM_OPT_TS)) { |
673 | /* | |
674 | * Configure timestamps to be emitted at regular intervals in | |
675 | * order to correlate instructions executed on different CPUs | |
676 | * (CPU-wide trace scenarios). | |
677 | */ | |
678 | ret = etm4_config_timestamp_event(drvdata); | |
679 | ||
680 | /* | |
681 | * No need to go further if timestamp intervals can't | |
682 | * be configured. | |
683 | */ | |
684 | if (ret) | |
685 | goto out; | |
686 | ||
27a7e2a7 | 687 | /* bit[11], Global timestamp tracing bit */ |
1cf50f64 | 688 | config->cfg |= TRCCONFIGR_TS; |
a54e14f8 | 689 | } |
82500a81 | 690 | |
aab47386 LY |
691 | /* Only trace contextID when runs in root PID namespace */ |
692 | if ((attr->config & BIT(ETM_OPT_CTXTID)) && | |
693 | task_is_in_init_pid_ns(current)) | |
82500a81 | 694 | /* bit[6], Context ID tracing bit */ |
1cf50f64 | 695 | config->cfg |= TRCCONFIGR_CID; |
82500a81 | 696 | |
88f11864 SP |
697 | /* |
698 | * If set bit ETM_OPT_CTXTID2 in perf config, this asks to trace VMID | |
699 | * for recording CONTEXTIDR_EL2. Do not enable VMID tracing if the | |
700 | * kernel is not running in EL2. | |
701 | */ | |
702 | if (attr->config & BIT(ETM_OPT_CTXTID2)) { | |
703 | if (!is_kernel_in_hyp_mode()) { | |
704 | ret = -EINVAL; | |
705 | goto out; | |
706 | } | |
aab47386 LY |
707 | /* Only trace virtual contextID when runs in root PID namespace */ |
708 | if (task_is_in_init_pid_ns(current)) | |
1cf50f64 | 709 | config->cfg |= TRCCONFIGR_VMID | TRCCONFIGR_VMIDOPT; |
88f11864 SP |
710 | } |
711 | ||
27b8f667 ML |
712 | /* return stack - enable if selected and supported */ |
713 | if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack) | |
714 | /* bit[12], Return stack enable bit */ | |
1cf50f64 | 715 | config->cfg |= TRCCONFIGR_RS; |
37fbbdbd | 716 | |
810ac401 ML |
717 | /* |
718 | * Set any selected configuration and preset. | |
719 | * | |
720 | * This extracts the values of PMU_FORMAT_ATTR(configid) and PMU_FORMAT_ATTR(preset) | |
721 | * in the perf attributes defined in coresight-etm-perf.c. | |
722 | * configid uses bits 63:32 of attr->config2, preset uses bits 3:0 of attr->config. | |
723 | * A zero configid means no configuration active, preset = 0 means no preset selected. | |
724 | */ | |
725 | if (attr->config2 & GENMASK_ULL(63, 32)) { | |
726 | cfg_hash = (u32)(attr->config2 >> 32); | |
727 | preset = attr->config & 0xF; | |
728 | ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); | |
729 | } | |
730 | ||
2d693ed4 JC |
731 | /* branch broadcast - enable if selected and supported */ |
732 | if (attr->config & BIT(ETM_OPT_BRANCH_BROADCAST)) { | |
733 | if (!drvdata->trcbb) { | |
734 | /* | |
735 | * Missing BB support could cause silent decode errors | |
736 | * so fail to open if it's not supported. | |
737 | */ | |
738 | ret = -EINVAL; | |
739 | goto out; | |
740 | } else { | |
741 | config->cfg |= BIT(ETM4_CFG_BIT_BB); | |
742 | } | |
743 | } | |
744 | ||
2703d74c MP |
745 | out: |
746 | return ret; | |
37fbbdbd MP |
747 | } |
748 | ||
749 | static int etm4_enable_perf(struct coresight_device *csdev, | |
68905d73 | 750 | struct perf_event *event) |
37fbbdbd | 751 | { |
df487120 | 752 | int ret = 0, trace_id; |
37fbbdbd MP |
753 | struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); |
754 | ||
2703d74c MP |
755 | if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) { |
756 | ret = -EINVAL; | |
757 | goto out; | |
758 | } | |
37fbbdbd MP |
759 | |
760 | /* Configure the tracer based on the session's specifics */ | |
810ac401 | 761 | ret = etm4_parse_event_config(csdev, event); |
2703d74c MP |
762 | if (ret) |
763 | goto out; | |
df487120 ML |
764 | |
765 | /* | |
766 | * perf allocates cpu ids as part of _setup_aux() - device needs to use | |
767 | * the allocated ID. This reads the current version without allocation. | |
768 | * | |
769 | * This does not use the trace id lock to prevent lock_dep issues | |
770 | * with perf locks - we know the ID cannot change until perf shuts down | |
771 | * the session | |
772 | */ | |
773 | trace_id = coresight_trace_id_read_cpu_id(drvdata->cpu); | |
774 | if (!IS_VALID_CS_TRACE_ID(trace_id)) { | |
775 | dev_err(&drvdata->csdev->dev, "Failed to set trace ID for %s on CPU%d\n", | |
776 | dev_name(&drvdata->csdev->dev), drvdata->cpu); | |
777 | ret = -EINVAL; | |
778 | goto out; | |
779 | } | |
780 | drvdata->trcid = (u8)trace_id; | |
781 | ||
37fbbdbd | 782 | /* And enable it */ |
e006d89a | 783 | ret = etm4_enable_hw(drvdata); |
37fbbdbd | 784 | |
2703d74c MP |
785 | out: |
786 | return ret; | |
37fbbdbd MP |
787 | } |
788 | ||
c38a9ec2 | 789 | static int etm4_enable_sysfs(struct coresight_device *csdev) |
2e1cdfe1 PP |
790 | { |
791 | struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); | |
bab223f9 | 792 | struct etm4_enable_arg arg = { }; |
7ebd0ec6 ML |
793 | unsigned long cfg_hash; |
794 | int ret, preset; | |
795 | ||
796 | /* enable any config activated by configfs */ | |
797 | cscfg_config_sysfs_get_active_cfg(&cfg_hash, &preset); | |
798 | if (cfg_hash) { | |
799 | ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); | |
800 | if (ret) | |
801 | return ret; | |
802 | } | |
2e1cdfe1 | 803 | |
2e1cdfe1 PP |
804 | spin_lock(&drvdata->spinlock); |
805 | ||
df487120 ML |
806 | /* sysfs needs to read and allocate a trace ID */ |
807 | ret = etm4_read_alloc_trace_id(drvdata); | |
808 | if (ret < 0) | |
809 | goto unlock_sysfs_enable; | |
810 | ||
2e1cdfe1 PP |
811 | /* |
812 | * Executing etm4_enable_hw on the cpu whose ETM is being enabled | |
813 | * ensures that register writes occur when cpu is powered. | |
814 | */ | |
e006d89a | 815 | arg.drvdata = drvdata; |
2e1cdfe1 | 816 | ret = smp_call_function_single(drvdata->cpu, |
e006d89a SP |
817 | etm4_enable_hw_smp_call, &arg, 1); |
818 | if (!ret) | |
819 | ret = arg.rc; | |
820 | if (!ret) | |
821 | drvdata->sticky_enable = true; | |
df487120 ML |
822 | |
823 | if (ret) | |
824 | etm4_release_trace_id(drvdata); | |
825 | ||
826 | unlock_sysfs_enable: | |
2e1cdfe1 PP |
827 | spin_unlock(&drvdata->spinlock); |
828 | ||
e006d89a | 829 | if (!ret) |
aaff7623 | 830 | dev_dbg(&csdev->dev, "ETM tracing enabled\n"); |
2e1cdfe1 PP |
831 | return ret; |
832 | } | |
833 | ||
9fa36828 JC |
834 | static int etm4_enable(struct coresight_device *csdev, struct perf_event *event, |
835 | enum cs_mode mode) | |
c38a9ec2 MP |
836 | { |
837 | int ret; | |
838 | u32 val; | |
839 | struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); | |
840 | ||
841 | val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode); | |
842 | ||
843 | /* Someone is already using the tracer */ | |
844 | if (val) | |
845 | return -EBUSY; | |
846 | ||
847 | switch (mode) { | |
848 | case CS_MODE_SYSFS: | |
849 | ret = etm4_enable_sysfs(csdev); | |
850 | break; | |
37fbbdbd | 851 | case CS_MODE_PERF: |
68905d73 | 852 | ret = etm4_enable_perf(csdev, event); |
37fbbdbd | 853 | break; |
c38a9ec2 MP |
854 | default: |
855 | ret = -EINVAL; | |
856 | } | |
857 | ||
858 | /* The tracer didn't start */ | |
859 | if (ret) | |
860 | local_set(&drvdata->mode, CS_MODE_DISABLED); | |
861 | ||
862 | return ret; | |
863 | } | |
864 | ||
2e1cdfe1 PP |
865 | static void etm4_disable_hw(void *info) |
866 | { | |
867 | u32 control; | |
868 | struct etmv4_drvdata *drvdata = info; | |
ebddaad0 | 869 | struct etmv4_config *config = &drvdata->config; |
02005282 SP |
870 | struct coresight_device *csdev = drvdata->csdev; |
871 | struct device *etm_dev = &csdev->dev; | |
872 | struct csdev_access *csa = &csdev->access; | |
ebddaad0 | 873 | int i; |
2e1cdfe1 | 874 | |
33d5573a | 875 | etm4_cs_unlock(drvdata, csa); |
e7255092 | 876 | etm4_disable_arch_specific(drvdata); |
2e1cdfe1 | 877 | |
02510a5a TZ |
878 | if (!drvdata->skip_power_up) { |
879 | /* power can be removed from the trace unit now */ | |
f5bd5236 | 880 | control = etm4x_relaxed_read32(csa, TRCPDCR); |
02510a5a | 881 | control &= ~TRCPDCR_PU; |
f5bd5236 | 882 | etm4x_relaxed_write32(csa, control, TRCPDCR); |
02510a5a | 883 | } |
46a3d5cd | 884 | |
f5bd5236 | 885 | control = etm4x_relaxed_read32(csa, TRCPRGCTLR); |
2e1cdfe1 PP |
886 | |
887 | /* EN, bit[0] Trace unit enable bit */ | |
888 | control &= ~0x1; | |
889 | ||
8b481196 SP |
890 | /* |
891 | * If the CPU supports v8.4 Trace filter Control, | |
892 | * set the ETM to trace prohibited region. | |
893 | */ | |
5f6fd1aa | 894 | etm4x_prohibit_trace(drvdata); |
1004ce4c AM |
895 | /* |
896 | * Make sure everything completes before disabling, as recommended | |
897 | * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register, | |
898 | * SSTATUS") of ARM IHI 0064D | |
899 | */ | |
900 | dsb(sy); | |
2e1cdfe1 | 901 | isb(); |
8b481196 SP |
902 | /* Trace synchronization barrier, is a nop if not supported */ |
903 | tsb_csync(); | |
f5bd5236 | 904 | etm4x_relaxed_write32(csa, control, TRCPRGCTLR); |
2e1cdfe1 | 905 | |
ebddaad0 | 906 | /* wait for TRCSTATR.PMSTABLE to go to '1' */ |
02005282 | 907 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) |
ebddaad0 ML |
908 | dev_err(etm_dev, |
909 | "timeout while waiting for PM stable Trace Status\n"); | |
ebddaad0 ML |
910 | /* read the status of the single shot comparators */ |
911 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { | |
912 | config->ss_status[i] = | |
f5bd5236 | 913 | etm4x_relaxed_read32(csa, TRCSSCSRn(i)); |
ebddaad0 ML |
914 | } |
915 | ||
8fa43700 ML |
916 | /* read back the current counter values */ |
917 | for (i = 0; i < drvdata->nr_cntr; i++) { | |
918 | config->cntr_val[i] = | |
f5bd5236 | 919 | etm4x_relaxed_read32(csa, TRCCNTVRn(i)); |
8fa43700 ML |
920 | } |
921 | ||
8ce00296 | 922 | coresight_disclaim_device_unlocked(csdev); |
33d5573a | 923 | etm4_cs_lock(drvdata, csa); |
2e1cdfe1 | 924 | |
aaff7623 SP |
925 | dev_dbg(&drvdata->csdev->dev, |
926 | "cpu: %d disable smp call done\n", drvdata->cpu); | |
2e1cdfe1 PP |
927 | } |
928 | ||
68905d73 MP |
929 | static int etm4_disable_perf(struct coresight_device *csdev, |
930 | struct perf_event *event) | |
37fbbdbd | 931 | { |
e97b1c6a MP |
932 | u32 control; |
933 | struct etm_filters *filters = event->hw.addr_filters; | |
37fbbdbd | 934 | struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); |
810ac401 | 935 | struct perf_event_attr *attr = &event->attr; |
37fbbdbd MP |
936 | |
937 | if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) | |
938 | return -EINVAL; | |
939 | ||
940 | etm4_disable_hw(drvdata); | |
810ac401 ML |
941 | /* |
942 | * The config_id occupies bits 63:32 of the config2 perf event attr | |
943 | * field. If this is non-zero then we will have enabled a config. | |
944 | */ | |
945 | if (attr->config2 & GENMASK_ULL(63, 32)) | |
946 | cscfg_csdev_disable_active_config(csdev); | |
e97b1c6a MP |
947 | |
948 | /* | |
949 | * Check if the start/stop logic was active when the unit was stopped. | |
950 | * That way we can re-enable the start/stop logic when the process is | |
951 | * scheduled again. Configuration of the start/stop logic happens in | |
952 | * function etm4_set_event_filters(). | |
953 | */ | |
f5bd5236 | 954 | control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR); |
e97b1c6a MP |
955 | /* TRCVICTLR::SSSTATUS, bit[9] */ |
956 | filters->ssstatus = (control & BIT(9)); | |
957 | ||
df487120 ML |
958 | /* |
959 | * perf will release trace ids when _free_aux() is | |
960 | * called at the end of the session. | |
961 | */ | |
962 | ||
37fbbdbd MP |
963 | return 0; |
964 | } | |
965 | ||
c38a9ec2 | 966 | static void etm4_disable_sysfs(struct coresight_device *csdev) |
2e1cdfe1 PP |
967 | { |
968 | struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); | |
969 | ||
970 | /* | |
971 | * Taking hotplug lock here protects from clocks getting disabled | |
972 | * with tracing being left on (crash scenario) if user disable occurs | |
973 | * after cpu online mask indicates the cpu is offline but before the | |
974 | * DYING hotplug callback is serviced by the ETM driver. | |
975 | */ | |
e9f5d63f | 976 | cpus_read_lock(); |
2e1cdfe1 PP |
977 | spin_lock(&drvdata->spinlock); |
978 | ||
979 | /* | |
980 | * Executing etm4_disable_hw on the cpu whose ETM is being disabled | |
981 | * ensures that register writes occur when cpu is powered. | |
982 | */ | |
983 | smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1); | |
2e1cdfe1 PP |
984 | |
985 | spin_unlock(&drvdata->spinlock); | |
e9f5d63f | 986 | cpus_read_unlock(); |
2e1cdfe1 | 987 | |
df487120 ML |
988 | /* |
989 | * we only release trace IDs when resetting sysfs. | |
990 | * This permits sysfs users to read the trace ID after the trace | |
991 | * session has completed. This maintains operational behaviour with | |
992 | * prior trace id allocation method | |
993 | */ | |
994 | ||
aaff7623 | 995 | dev_dbg(&csdev->dev, "ETM tracing disabled\n"); |
2e1cdfe1 PP |
996 | } |
997 | ||
68905d73 MP |
998 | static void etm4_disable(struct coresight_device *csdev, |
999 | struct perf_event *event) | |
c38a9ec2 | 1000 | { |
9fa36828 | 1001 | enum cs_mode mode; |
c38a9ec2 MP |
1002 | struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); |
1003 | ||
1004 | /* | |
1005 | * For as long as the tracer isn't disabled another entity can't | |
1006 | * change its status. As such we can read the status here without | |
1007 | * fearing it will change under us. | |
1008 | */ | |
1009 | mode = local_read(&drvdata->mode); | |
1010 | ||
1011 | switch (mode) { | |
1012 | case CS_MODE_DISABLED: | |
1013 | break; | |
1014 | case CS_MODE_SYSFS: | |
1015 | etm4_disable_sysfs(csdev); | |
1016 | break; | |
37fbbdbd | 1017 | case CS_MODE_PERF: |
68905d73 | 1018 | etm4_disable_perf(csdev, event); |
37fbbdbd | 1019 | break; |
c38a9ec2 MP |
1020 | } |
1021 | ||
1022 | if (mode) | |
1023 | local_set(&drvdata->mode, CS_MODE_DISABLED); | |
1024 | } | |
1025 | ||
2e1cdfe1 | 1026 | static const struct coresight_ops_source etm4_source_ops = { |
52210c87 | 1027 | .cpu_id = etm4_cpu_id, |
2e1cdfe1 PP |
1028 | .enable = etm4_enable, |
1029 | .disable = etm4_disable, | |
1030 | }; | |
1031 | ||
1032 | static const struct coresight_ops etm4_cs_ops = { | |
1033 | .source_ops = &etm4_source_ops, | |
1034 | }; | |
1035 | ||
dc1747a7 SP |
1036 | static inline bool cpu_supports_sysreg_trace(void) |
1037 | { | |
1038 | u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); | |
1039 | ||
db74cd63 | 1040 | return ((dfr0 >> ID_AA64DFR0_EL1_TraceVer_SHIFT) & 0xfUL) > 0; |
dc1747a7 SP |
1041 | } |
1042 | ||
1043 | static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata, | |
1044 | struct csdev_access *csa) | |
1045 | { | |
1046 | u32 devarch; | |
1047 | ||
1048 | if (!cpu_supports_sysreg_trace()) | |
1049 | return false; | |
1050 | ||
1051 | /* | |
1052 | * ETMs implementing sysreg access must implement TRCDEVARCH. | |
1053 | */ | |
1054 | devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH); | |
35e1c916 SP |
1055 | switch (devarch & ETM_DEVARCH_ID_MASK) { |
1056 | case ETM_DEVARCH_ETMv4x_ARCH: | |
1057 | *csa = (struct csdev_access) { | |
1058 | .io_mem = false, | |
1059 | .read = etm4x_sysreg_read, | |
1060 | .write = etm4x_sysreg_write, | |
1061 | }; | |
1062 | break; | |
1063 | case ETM_DEVARCH_ETE_ARCH: | |
1064 | *csa = (struct csdev_access) { | |
1065 | .io_mem = false, | |
1066 | .read = ete_sysreg_read, | |
1067 | .write = ete_sysreg_write, | |
1068 | }; | |
1069 | break; | |
1070 | default: | |
dc1747a7 | 1071 | return false; |
35e1c916 | 1072 | } |
dc1747a7 SP |
1073 | |
1074 | drvdata->arch = etm_devarch_to_arch(devarch); | |
1075 | return true; | |
1076 | } | |
1077 | ||
73d779a0 AK |
1078 | static bool is_devtype_cpu_trace(void __iomem *base) |
1079 | { | |
1080 | u32 devtype = readl(base + TRCDEVTYPE); | |
1081 | ||
1082 | return (devtype == CS_DEVTYPE_PE_TRACE); | |
1083 | } | |
1084 | ||
fd6e7905 SP |
1085 | static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, |
1086 | struct csdev_access *csa) | |
1087 | { | |
8b94db1e | 1088 | u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH); |
8b94db1e | 1089 | |
73d779a0 AK |
1090 | if (!is_coresight_device(drvdata->base) || !is_devtype_cpu_trace(drvdata->base)) |
1091 | return false; | |
1092 | ||
8b94db1e SP |
1093 | /* |
1094 | * All ETMs must implement TRCDEVARCH to indicate that | |
735e7b30 SP |
1095 | * the component is an ETMv4. Even though TRCIDR1 also |
1096 | * contains the information, it is part of the "Trace" | |
1097 | * register and must be accessed with the OSLK cleared, | |
1098 | * with MMIO. But we cannot touch the OSLK until we are | |
1099 | * sure this is an ETM. So rely only on the TRCDEVARCH. | |
8b94db1e | 1100 | */ |
735e7b30 SP |
1101 | if ((devarch & ETM_DEVARCH_ID_MASK) != ETM_DEVARCH_ETMv4x_ARCH) { |
1102 | pr_warn_once("TRCDEVARCH doesn't match ETMv4 architecture\n"); | |
1103 | return false; | |
8b94db1e SP |
1104 | } |
1105 | ||
735e7b30 | 1106 | drvdata->arch = etm_devarch_to_arch(devarch); |
fd6e7905 SP |
1107 | *csa = CSDEV_ACCESS_IOMEM(drvdata->base); |
1108 | return true; | |
1109 | } | |
1110 | ||
1111 | static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, | |
1112 | struct csdev_access *csa) | |
1113 | { | |
dc1747a7 SP |
1114 | /* |
1115 | * Always choose the memory mapped io, if there is | |
1116 | * a memory map to prevent sysreg access on broken | |
1117 | * systems. | |
1118 | */ | |
fd6e7905 SP |
1119 | if (drvdata->base) |
1120 | return etm4_init_iomem_access(drvdata, csa); | |
1121 | ||
dc1747a7 SP |
1122 | if (etm4_init_sysreg_access(drvdata, csa)) |
1123 | return true; | |
1124 | ||
fd6e7905 SP |
1125 | return false; |
1126 | } | |
1127 | ||
5f6fd1aa | 1128 | static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata) |
e5d51fbe JZ |
1129 | { |
1130 | u64 dfr0 = read_sysreg(id_aa64dfr0_el1); | |
1131 | u64 trfcr; | |
1132 | ||
5f6fd1aa | 1133 | drvdata->trfcr = 0; |
db74cd63 | 1134 | if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceFilt_SHIFT)) |
e5d51fbe JZ |
1135 | return; |
1136 | ||
1137 | /* | |
1138 | * If the CPU supports v8.4 SelfHosted Tracing, enable | |
1139 | * tracing at the kernel EL and EL0, forcing to use the | |
1140 | * virtual time as the timestamp. | |
1141 | */ | |
1142 | trfcr = (TRFCR_ELx_TS_VIRTUAL | | |
1143 | TRFCR_ELx_ExTRE | | |
1144 | TRFCR_ELx_E0TRE); | |
1145 | ||
1146 | /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */ | |
1147 | if (is_kernel_in_hyp_mode()) | |
1148 | trfcr |= TRFCR_EL2_CX; | |
1149 | ||
5f6fd1aa | 1150 | drvdata->trfcr = trfcr; |
e5d51fbe JZ |
1151 | } |
1152 | ||
2e1cdfe1 PP |
1153 | static void etm4_init_arch_data(void *info) |
1154 | { | |
1155 | u32 etmidr0; | |
2e1cdfe1 PP |
1156 | u32 etmidr2; |
1157 | u32 etmidr3; | |
1158 | u32 etmidr4; | |
1159 | u32 etmidr5; | |
fd6e7905 SP |
1160 | struct etm4_init_arg *init_arg = info; |
1161 | struct etmv4_drvdata *drvdata; | |
1162 | struct csdev_access *csa; | |
ebddaad0 | 1163 | int i; |
2e1cdfe1 | 1164 | |
3c728e07 | 1165 | drvdata = dev_get_drvdata(init_arg->dev); |
fd6e7905 SP |
1166 | csa = init_arg->csa; |
1167 | ||
1168 | /* | |
1169 | * If we are unable to detect the access mechanism, | |
1170 | * or unable to detect the trace unit type, fail | |
1171 | * early. | |
1172 | */ | |
1173 | if (!etm4_init_csdev_access(drvdata, csa)) | |
1174 | return; | |
1175 | ||
bc2c689f SP |
1176 | /* Detect the support for OS Lock before we actually use it */ |
1177 | etm_detect_os_lock(drvdata, csa); | |
1178 | ||
66bbbb77 | 1179 | /* Make sure all registers are accessible */ |
f5bd5236 | 1180 | etm4_os_unlock_csa(drvdata, csa); |
33d5573a | 1181 | etm4_cs_unlock(drvdata, csa); |
2e1cdfe1 | 1182 | |
5a1c7097 | 1183 | etm4_check_arch_features(drvdata, csa); |
e97db2cf | 1184 | |
2e1cdfe1 | 1185 | /* find all capabilities of the tracing unit */ |
f5bd5236 | 1186 | etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0); |
2e1cdfe1 PP |
1187 | |
1188 | /* INSTP0, bits[2:1] P0 tracing support field */ | |
e601cc9a | 1189 | drvdata->instrp0 = !!(FIELD_GET(TRCIDR0_INSTP0_MASK, etmidr0) == 0b11); |
2e1cdfe1 | 1190 | /* TRCBB, bit[5] Branch broadcast tracing support bit */ |
e601cc9a | 1191 | drvdata->trcbb = !!(etmidr0 & TRCIDR0_TRCBB); |
2e1cdfe1 | 1192 | /* TRCCOND, bit[6] Conditional instruction tracing support bit */ |
e601cc9a | 1193 | drvdata->trccond = !!(etmidr0 & TRCIDR0_TRCCOND); |
2e1cdfe1 | 1194 | /* TRCCCI, bit[7] Cycle counting instruction bit */ |
e601cc9a | 1195 | drvdata->trccci = !!(etmidr0 & TRCIDR0_TRCCCI); |
2e1cdfe1 | 1196 | /* RETSTACK, bit[9] Return stack bit */ |
e601cc9a | 1197 | drvdata->retstack = !!(etmidr0 & TRCIDR0_RETSTACK); |
2e1cdfe1 | 1198 | /* NUMEVENT, bits[11:10] Number of events field */ |
e601cc9a | 1199 | drvdata->nr_event = FIELD_GET(TRCIDR0_NUMEVENT_MASK, etmidr0); |
2e1cdfe1 | 1200 | /* QSUPP, bits[16:15] Q element support field */ |
e601cc9a | 1201 | drvdata->q_support = FIELD_GET(TRCIDR0_QSUPP_MASK, etmidr0); |
2e1cdfe1 | 1202 | /* TSSIZE, bits[28:24] Global timestamp size field */ |
e601cc9a | 1203 | drvdata->ts_size = FIELD_GET(TRCIDR0_TSSIZE_MASK, etmidr0); |
2e1cdfe1 | 1204 | |
2e1cdfe1 | 1205 | /* maximum size of resources */ |
f5bd5236 | 1206 | etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2); |
2e1cdfe1 | 1207 | /* CIDSIZE, bits[9:5] Indicates the Context ID size */ |
cf0c7f18 | 1208 | drvdata->ctxid_size = FIELD_GET(TRCIDR2_CIDSIZE_MASK, etmidr2); |
2e1cdfe1 | 1209 | /* VMIDSIZE, bits[14:10] Indicates the VMID size */ |
cf0c7f18 | 1210 | drvdata->vmid_size = FIELD_GET(TRCIDR2_VMIDSIZE_MASK, etmidr2); |
2e1cdfe1 | 1211 | /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */ |
cf0c7f18 | 1212 | drvdata->ccsize = FIELD_GET(TRCIDR2_CCSIZE_MASK, etmidr2); |
2e1cdfe1 | 1213 | |
f5bd5236 | 1214 | etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3); |
2e1cdfe1 | 1215 | /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */ |
f4d1f214 | 1216 | drvdata->ccitmin = FIELD_GET(TRCIDR3_CCITMIN_MASK, etmidr3); |
2e1cdfe1 | 1217 | /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */ |
f4d1f214 | 1218 | drvdata->s_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_S_MASK, etmidr3); |
1d3eead7 | 1219 | drvdata->config.s_ex_level = drvdata->s_ex_level; |
2e1cdfe1 | 1220 | /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */ |
f4d1f214 | 1221 | drvdata->ns_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_NS_MASK, etmidr3); |
2e1cdfe1 PP |
1222 | /* |
1223 | * TRCERR, bit[24] whether a trace unit can trace a | |
1224 | * system error exception. | |
1225 | */ | |
f4d1f214 | 1226 | drvdata->trc_error = !!(etmidr3 & TRCIDR3_TRCERR); |
2e1cdfe1 | 1227 | /* SYNCPR, bit[25] implementation has a fixed synchronization period? */ |
f4d1f214 | 1228 | drvdata->syncpr = !!(etmidr3 & TRCIDR3_SYNCPR); |
2e1cdfe1 | 1229 | /* STALLCTL, bit[26] is stall control implemented? */ |
f4d1f214 | 1230 | drvdata->stallctl = !!(etmidr3 & TRCIDR3_STALLCTL); |
2e1cdfe1 | 1231 | /* SYSSTALL, bit[27] implementation can support stall control? */ |
f4d1f214 | 1232 | drvdata->sysstall = !!(etmidr3 & TRCIDR3_SYSSTALL); |
4e218727 SP |
1233 | /* |
1234 | * NUMPROC - the number of PEs available for tracing, 5bits | |
1235 | * = TRCIDR3.bits[13:12]bits[30:28] | |
1236 | * bits[4:3] = TRCIDR3.bits[13:12] (since etm-v4.2, otherwise RES0) | |
1237 | * bits[3:0] = TRCIDR3.bits[30:28] | |
1238 | */ | |
f4d1f214 JC |
1239 | drvdata->nr_pe = (FIELD_GET(TRCIDR3_NUMPROC_HI_MASK, etmidr3) << 3) | |
1240 | FIELD_GET(TRCIDR3_NUMPROC_LO_MASK, etmidr3); | |
2e1cdfe1 | 1241 | /* NOOVERFLOW, bit[31] is trace overflow prevention supported */ |
f4d1f214 | 1242 | drvdata->nooverflow = !!(etmidr3 & TRCIDR3_NOOVERFLOW); |
2e1cdfe1 PP |
1243 | |
1244 | /* number of resources trace unit supports */ | |
f5bd5236 | 1245 | etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4); |
2e1cdfe1 | 1246 | /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */ |
ea69dbb8 | 1247 | drvdata->nr_addr_cmp = FIELD_GET(TRCIDR4_NUMACPAIRS_MASK, etmidr4); |
2e1cdfe1 | 1248 | /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */ |
ea69dbb8 | 1249 | drvdata->nr_pe_cmp = FIELD_GET(TRCIDR4_NUMPC_MASK, etmidr4); |
497b5956 CZ |
1250 | /* |
1251 | * NUMRSPAIR, bits[19:16] | |
1252 | * The number of resource pairs conveyed by the HW starts at 0, i.e a | |
1253 | * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on. | |
1254 | * As such add 1 to the value of NUMRSPAIR for a better representation. | |
14ea4db1 ML |
1255 | * |
1256 | * For ETM v4.3 and later, 0x0 means 0, and no pairs are available - | |
1257 | * the default TRUE and FALSE resource selectors are omitted. | |
1258 | * Otherwise for values 0x1 and above the number is N + 1 as per v4.2. | |
497b5956 | 1259 | */ |
ea69dbb8 | 1260 | drvdata->nr_resource = FIELD_GET(TRCIDR4_NUMRSPAIR_MASK, etmidr4); |
e49516e2 | 1261 | if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0)) |
14ea4db1 | 1262 | drvdata->nr_resource += 1; |
2e1cdfe1 PP |
1263 | /* |
1264 | * NUMSSCC, bits[23:20] the number of single-shot | |
ebddaad0 ML |
1265 | * comparator control for tracing. Read any status regs as these |
1266 | * also contain RO capability data. | |
2e1cdfe1 | 1267 | */ |
ea69dbb8 | 1268 | drvdata->nr_ss_cmp = FIELD_GET(TRCIDR4_NUMSSCC_MASK, etmidr4); |
ebddaad0 ML |
1269 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { |
1270 | drvdata->config.ss_status[i] = | |
f5bd5236 | 1271 | etm4x_relaxed_read32(csa, TRCSSCSRn(i)); |
ebddaad0 | 1272 | } |
2e1cdfe1 | 1273 | /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */ |
ea69dbb8 | 1274 | drvdata->numcidc = FIELD_GET(TRCIDR4_NUMCIDC_MASK, etmidr4); |
2e1cdfe1 | 1275 | /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */ |
ea69dbb8 | 1276 | drvdata->numvmidc = FIELD_GET(TRCIDR4_NUMVMIDC_MASK, etmidr4); |
2e1cdfe1 | 1277 | |
f5bd5236 | 1278 | etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5); |
2e1cdfe1 | 1279 | /* NUMEXTIN, bits[8:0] number of external inputs implemented */ |
028e5460 | 1280 | drvdata->nr_ext_inp = FIELD_GET(TRCIDR5_NUMEXTIN_MASK, etmidr5); |
2e1cdfe1 | 1281 | /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */ |
028e5460 | 1282 | drvdata->trcid_size = FIELD_GET(TRCIDR5_TRACEIDSIZE_MASK, etmidr5); |
2e1cdfe1 | 1283 | /* ATBTRIG, bit[22] implementation can support ATB triggers? */ |
028e5460 | 1284 | drvdata->atbtrig = !!(etmidr5 & TRCIDR5_ATBTRIG); |
2e1cdfe1 PP |
1285 | /* |
1286 | * LPOVERRIDE, bit[23] implementation supports | |
1287 | * low-power state override | |
1288 | */ | |
028e5460 | 1289 | drvdata->lpoverride = (etmidr5 & TRCIDR5_LPOVERRIDE) && (!drvdata->skip_power_up); |
2e1cdfe1 | 1290 | /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */ |
028e5460 | 1291 | drvdata->nrseqstate = FIELD_GET(TRCIDR5_NUMSEQSTATE_MASK, etmidr5); |
2e1cdfe1 | 1292 | /* NUMCNTR, bits[30:28] number of counters available for tracing */ |
028e5460 | 1293 | drvdata->nr_cntr = FIELD_GET(TRCIDR5_NUMCNTR_MASK, etmidr5); |
33d5573a | 1294 | etm4_cs_lock(drvdata, csa); |
5f6fd1aa | 1295 | cpu_detect_trace_filtering(drvdata); |
2e1cdfe1 PP |
1296 | } |
1297 | ||
4d1b1fd7 SP |
1298 | static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config) |
1299 | { | |
6ba7f2bc | 1300 | return etm4_get_access_type(config) << __bf_shf(TRCVICTLR_EXLEVEL_MASK); |
4d1b1fd7 SP |
1301 | } |
1302 | ||
096dcfb9 ML |
1303 | /* Set ELx trace filter access in the TRCVICTLR register */ |
1304 | static void etm4_set_victlr_access(struct etmv4_config *config) | |
1305 | { | |
4d1b1fd7 SP |
1306 | config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK; |
1307 | config->vinst_ctrl |= etm4_get_victlr_access_type(config); | |
096dcfb9 ML |
1308 | } |
1309 | ||
2a5695a5 | 1310 | static void etm4_set_default_config(struct etmv4_config *config) |
2e1cdfe1 | 1311 | { |
2e1cdfe1 | 1312 | /* disable all events tracing */ |
54ff892b MP |
1313 | config->eventctrl0 = 0x0; |
1314 | config->eventctrl1 = 0x0; | |
2e1cdfe1 PP |
1315 | |
1316 | /* disable stalling */ | |
54ff892b | 1317 | config->stall_ctrl = 0x0; |
2e1cdfe1 | 1318 | |
fc208abe MP |
1319 | /* enable trace synchronization every 4096 bytes, if available */ |
1320 | config->syncfreq = 0xC; | |
1321 | ||
2e1cdfe1 | 1322 | /* disable timestamp event */ |
54ff892b | 1323 | config->ts_ctrl = 0x0; |
2e1cdfe1 | 1324 | |
fc208abe | 1325 | /* TRCVICTLR::EVENT = 0x01, select the always on logic */ |
6ba7f2bc | 1326 | config->vinst_ctrl = FIELD_PREP(TRCVICTLR_EVENT_MASK, 0x01); |
096dcfb9 ML |
1327 | |
1328 | /* TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering */ | |
1329 | etm4_set_victlr_access(config); | |
2a5695a5 | 1330 | } |
2e1cdfe1 | 1331 | |
b860801e | 1332 | static u64 etm4_get_ns_access_type(struct etmv4_config *config) |
2a5695a5 | 1333 | { |
6cccf663 | 1334 | u64 access_type = 0; |
f67b467a | 1335 | |
f0d30cc3 | 1336 | /* |
4d1b1fd7 SP |
1337 | * EXLEVEL_NS, for NonSecure Exception levels. |
1338 | * The mask here is a generic value and must be | |
1339 | * shifted to the corresponding field for the registers | |
f0d30cc3 | 1340 | */ |
b860801e TN |
1341 | if (!is_kernel_in_hyp_mode()) { |
1342 | /* Stay away from hypervisor mode for non-VHE */ | |
1343 | access_type = ETM_EXLEVEL_NS_HYP; | |
1344 | if (config->mode & ETM_MODE_EXCL_KERN) | |
1345 | access_type |= ETM_EXLEVEL_NS_OS; | |
1346 | } else if (config->mode & ETM_MODE_EXCL_KERN) { | |
1347 | access_type = ETM_EXLEVEL_NS_HYP; | |
1348 | } | |
f0d30cc3 MP |
1349 | |
1350 | if (config->mode & ETM_MODE_EXCL_USER) | |
1351 | access_type |= ETM_EXLEVEL_NS_APP; | |
1352 | ||
b860801e TN |
1353 | return access_type; |
1354 | } | |
1355 | ||
4d1b1fd7 SP |
1356 | /* |
1357 | * Construct the exception level masks for a given config. | |
1358 | * This must be shifted to the corresponding register field | |
1359 | * for usage. | |
1360 | */ | |
b860801e TN |
1361 | static u64 etm4_get_access_type(struct etmv4_config *config) |
1362 | { | |
4d1b1fd7 SP |
1363 | /* All Secure exception levels are excluded from the trace */ |
1364 | return etm4_get_ns_access_type(config) | (u64)config->s_ex_level; | |
1365 | } | |
5edd944b | 1366 | |
4d1b1fd7 SP |
1367 | static u64 etm4_get_comparator_access_type(struct etmv4_config *config) |
1368 | { | |
1369 | return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT; | |
f0d30cc3 MP |
1370 | } |
1371 | ||
1372 | static void etm4_set_comparator_filter(struct etmv4_config *config, | |
1373 | u64 start, u64 stop, int comparator) | |
1374 | { | |
4d1b1fd7 | 1375 | u64 access_type = etm4_get_comparator_access_type(config); |
f0d30cc3 | 1376 | |
5edd944b | 1377 | /* First half of default address comparator */ |
6cccf663 MP |
1378 | config->addr_val[comparator] = start; |
1379 | config->addr_acc[comparator] = access_type; | |
1380 | config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE; | |
5edd944b MP |
1381 | |
1382 | /* Second half of default address comparator */ | |
6cccf663 MP |
1383 | config->addr_val[comparator + 1] = stop; |
1384 | config->addr_acc[comparator + 1] = access_type; | |
1385 | config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE; | |
1386 | ||
1387 | /* | |
1388 | * Configure the ViewInst function to include this address range | |
1389 | * comparator. | |
1390 | * | |
1391 | * @comparator is divided by two since it is the index in the | |
1392 | * etmv4_config::addr_val array but register TRCVIIECTLR deals with | |
1393 | * address range comparator _pairs_. | |
1394 | * | |
1395 | * Therefore: | |
1396 | * index 0 -> compatator pair 0 | |
1397 | * index 2 -> comparator pair 1 | |
1398 | * index 4 -> comparator pair 2 | |
1399 | * ... | |
1400 | * index 14 -> comparator pair 7 | |
1401 | */ | |
1402 | config->viiectlr |= BIT(comparator / 2); | |
1403 | } | |
1404 | ||
e97b1c6a MP |
1405 | static void etm4_set_start_stop_filter(struct etmv4_config *config, |
1406 | u64 address, int comparator, | |
1407 | enum etm_addr_type type) | |
1408 | { | |
1409 | int shift; | |
4d1b1fd7 | 1410 | u64 access_type = etm4_get_comparator_access_type(config); |
e97b1c6a MP |
1411 | |
1412 | /* Configure the comparator */ | |
1413 | config->addr_val[comparator] = address; | |
1414 | config->addr_acc[comparator] = access_type; | |
1415 | config->addr_type[comparator] = type; | |
1416 | ||
1417 | /* | |
1418 | * Configure ViewInst Start-Stop control register. | |
1419 | * Addresses configured to start tracing go from bit 0 to n-1, | |
1420 | * while those configured to stop tracing from 16 to 16 + n-1. | |
1421 | */ | |
1422 | shift = (type == ETM_ADDR_TYPE_START ? 0 : 16); | |
1423 | config->vissctlr |= BIT(shift + comparator); | |
1424 | } | |
1425 | ||
6cccf663 MP |
1426 | static void etm4_set_default_filter(struct etmv4_config *config) |
1427 | { | |
ae204151 ML |
1428 | /* Trace everything 'default' filter achieved by no filtering */ |
1429 | config->viiectlr = 0x0; | |
fc208abe | 1430 | |
5edd944b MP |
1431 | /* |
1432 | * TRCVICTLR::SSSTATUS == 1, the start-stop logic is | |
1433 | * in the started state | |
1434 | */ | |
6ba7f2bc | 1435 | config->vinst_ctrl |= TRCVICTLR_SSSTATUS; |
1b6b0e08 | 1436 | config->mode |= ETM_MODE_VIEWINST_STARTSTOP; |
5edd944b MP |
1437 | |
1438 | /* No start-stop filtering for ViewInst */ | |
fc208abe | 1439 | config->vissctlr = 0x0; |
2e1cdfe1 PP |
1440 | } |
1441 | ||
2a5695a5 MP |
1442 | static void etm4_set_default(struct etmv4_config *config) |
1443 | { | |
1444 | if (WARN_ON_ONCE(!config)) | |
1445 | return; | |
1446 | ||
1447 | /* | |
1448 | * Make default initialisation trace everything | |
1449 | * | |
ae204151 ML |
1450 | * This is done by a minimum default config sufficient to enable |
1451 | * full instruction trace - with a default filter for trace all | |
1452 | * achieved by having no filtering. | |
2a5695a5 MP |
1453 | */ |
1454 | etm4_set_default_config(config); | |
1455 | etm4_set_default_filter(config); | |
1456 | } | |
1457 | ||
2703d74c MP |
1458 | static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type) |
1459 | { | |
1460 | int nr_comparator, index = 0; | |
1461 | struct etmv4_config *config = &drvdata->config; | |
1462 | ||
1463 | /* | |
1464 | * nr_addr_cmp holds the number of comparator _pair_, so time 2 | |
1465 | * for the total number of comparators. | |
1466 | */ | |
1467 | nr_comparator = drvdata->nr_addr_cmp * 2; | |
1468 | ||
1469 | /* Go through the tally of comparators looking for a free one. */ | |
1470 | while (index < nr_comparator) { | |
1471 | switch (type) { | |
1472 | case ETM_ADDR_TYPE_RANGE: | |
1473 | if (config->addr_type[index] == ETM_ADDR_TYPE_NONE && | |
1474 | config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE) | |
1475 | return index; | |
1476 | ||
1477 | /* Address range comparators go in pairs */ | |
1478 | index += 2; | |
1479 | break; | |
e97b1c6a MP |
1480 | case ETM_ADDR_TYPE_START: |
1481 | case ETM_ADDR_TYPE_STOP: | |
1482 | if (config->addr_type[index] == ETM_ADDR_TYPE_NONE) | |
1483 | return index; | |
1484 | ||
1485 | /* Start/stop address can have odd indexes */ | |
1486 | index += 1; | |
1487 | break; | |
2703d74c MP |
1488 | default: |
1489 | return -EINVAL; | |
1490 | } | |
1491 | } | |
1492 | ||
1493 | /* If we are here all the comparators have been used. */ | |
1494 | return -ENOSPC; | |
1495 | } | |
1496 | ||
1497 | static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, | |
1498 | struct perf_event *event) | |
1499 | { | |
1500 | int i, comparator, ret = 0; | |
e97b1c6a | 1501 | u64 address; |
2703d74c MP |
1502 | struct etmv4_config *config = &drvdata->config; |
1503 | struct etm_filters *filters = event->hw.addr_filters; | |
1504 | ||
1505 | if (!filters) | |
1506 | goto default_filter; | |
1507 | ||
1508 | /* Sync events with what Perf got */ | |
1509 | perf_event_addr_filters_sync(event); | |
1510 | ||
1511 | /* | |
1512 | * If there are no filters to deal with simply go ahead with | |
1513 | * the default filter, i.e the entire address range. | |
1514 | */ | |
1515 | if (!filters->nr_filters) | |
1516 | goto default_filter; | |
1517 | ||
1518 | for (i = 0; i < filters->nr_filters; i++) { | |
1519 | struct etm_filter *filter = &filters->etm_filter[i]; | |
1520 | enum etm_addr_type type = filter->type; | |
1521 | ||
1522 | /* See if a comparator is free. */ | |
1523 | comparator = etm4_get_next_comparator(drvdata, type); | |
1524 | if (comparator < 0) { | |
1525 | ret = comparator; | |
1526 | goto out; | |
1527 | } | |
1528 | ||
1529 | switch (type) { | |
1530 | case ETM_ADDR_TYPE_RANGE: | |
1531 | etm4_set_comparator_filter(config, | |
1532 | filter->start_addr, | |
1533 | filter->stop_addr, | |
1534 | comparator); | |
1535 | /* | |
1536 | * TRCVICTLR::SSSTATUS == 1, the start-stop logic is | |
1537 | * in the started state | |
1538 | */ | |
6ba7f2bc | 1539 | config->vinst_ctrl |= TRCVICTLR_SSSTATUS; |
2703d74c MP |
1540 | |
1541 | /* No start-stop filtering for ViewInst */ | |
1542 | config->vissctlr = 0x0; | |
1543 | break; | |
e97b1c6a MP |
1544 | case ETM_ADDR_TYPE_START: |
1545 | case ETM_ADDR_TYPE_STOP: | |
1546 | /* Get the right start or stop address */ | |
1547 | address = (type == ETM_ADDR_TYPE_START ? | |
1548 | filter->start_addr : | |
1549 | filter->stop_addr); | |
1550 | ||
1551 | /* Configure comparator */ | |
1552 | etm4_set_start_stop_filter(config, address, | |
1553 | comparator, type); | |
1554 | ||
1555 | /* | |
1556 | * If filters::ssstatus == 1, trace acquisition was | |
1557 | * started but the process was yanked away before the | |
c767c347 | 1558 | * stop address was hit. As such the start/stop |
e97b1c6a MP |
1559 | * logic needs to be re-started so that tracing can |
1560 | * resume where it left. | |
1561 | * | |
1562 | * The start/stop logic status when a process is | |
1563 | * scheduled out is checked in function | |
1564 | * etm4_disable_perf(). | |
1565 | */ | |
1566 | if (filters->ssstatus) | |
6ba7f2bc | 1567 | config->vinst_ctrl |= TRCVICTLR_SSSTATUS; |
e97b1c6a MP |
1568 | |
1569 | /* No include/exclude filtering for ViewInst */ | |
1570 | config->viiectlr = 0x0; | |
1571 | break; | |
2703d74c MP |
1572 | default: |
1573 | ret = -EINVAL; | |
1574 | goto out; | |
1575 | } | |
1576 | } | |
1577 | ||
1578 | goto out; | |
1579 | ||
1580 | ||
1581 | default_filter: | |
1582 | etm4_set_default_filter(config); | |
1583 | ||
1584 | out: | |
1585 | return ret; | |
1586 | } | |
1587 | ||
4f6fce54 MP |
1588 | void etm4_config_trace_mode(struct etmv4_config *config) |
1589 | { | |
096dcfb9 | 1590 | u32 mode; |
4f6fce54 MP |
1591 | |
1592 | mode = config->mode; | |
1593 | mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER); | |
1594 | ||
1595 | /* excluding kernel AND user space doesn't make sense */ | |
1596 | WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER)); | |
1597 | ||
1598 | /* nothing to do if neither flags are set */ | |
1599 | if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER)) | |
1600 | return; | |
1601 | ||
096dcfb9 | 1602 | etm4_set_victlr_access(config); |
4f6fce54 MP |
1603 | } |
1604 | ||
58eb457b | 1605 | static int etm4_online_cpu(unsigned int cpu) |
2e1cdfe1 | 1606 | { |
2e1cdfe1 | 1607 | if (!etmdrvdata[cpu]) |
3c728e07 | 1608 | return etm4_probe_cpu(cpu); |
2e1cdfe1 | 1609 | |
58eb457b SAS |
1610 | if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable) |
1611 | coresight_enable(etmdrvdata[cpu]->csdev); | |
1612 | return 0; | |
1613 | } | |
2e1cdfe1 | 1614 | |
58eb457b SAS |
1615 | static int etm4_starting_cpu(unsigned int cpu) |
1616 | { | |
1617 | if (!etmdrvdata[cpu]) | |
1618 | return 0; | |
1619 | ||
1620 | spin_lock(&etmdrvdata[cpu]->spinlock); | |
6d765101 | 1621 | if (!etmdrvdata[cpu]->os_unlock) |
58eb457b | 1622 | etm4_os_unlock(etmdrvdata[cpu]); |
58eb457b SAS |
1623 | |
1624 | if (local_read(&etmdrvdata[cpu]->mode)) | |
1625 | etm4_enable_hw(etmdrvdata[cpu]); | |
1626 | spin_unlock(&etmdrvdata[cpu]->spinlock); | |
1627 | return 0; | |
2e1cdfe1 PP |
1628 | } |
1629 | ||
58eb457b SAS |
1630 | static int etm4_dying_cpu(unsigned int cpu) |
1631 | { | |
1632 | if (!etmdrvdata[cpu]) | |
1633 | return 0; | |
1634 | ||
1635 | spin_lock(&etmdrvdata[cpu]->spinlock); | |
1636 | if (local_read(&etmdrvdata[cpu]->mode)) | |
1637 | etm4_disable_hw(etmdrvdata[cpu]); | |
1638 | spin_unlock(&etmdrvdata[cpu]->spinlock); | |
1639 | return 0; | |
1640 | } | |
2e1cdfe1 | 1641 | |
937d3f58 | 1642 | static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) |
f188b5e7 AM |
1643 | { |
1644 | int i, ret = 0; | |
1645 | struct etmv4_save_state *state; | |
02005282 SP |
1646 | struct coresight_device *csdev = drvdata->csdev; |
1647 | struct csdev_access *csa; | |
1648 | struct device *etm_dev; | |
1649 | ||
1650 | if (WARN_ON(!csdev)) | |
1651 | return -ENODEV; | |
1652 | ||
1653 | etm_dev = &csdev->dev; | |
1654 | csa = &csdev->access; | |
f188b5e7 AM |
1655 | |
1656 | /* | |
1657 | * As recommended by 3.4.1 ("The procedure when powering down the PE") | |
1658 | * of ARM IHI 0064D | |
1659 | */ | |
1660 | dsb(sy); | |
1661 | isb(); | |
1662 | ||
33d5573a | 1663 | etm4_cs_unlock(drvdata, csa); |
f188b5e7 AM |
1664 | /* Lock the OS lock to disable trace and external debugger access */ |
1665 | etm4_os_lock(drvdata); | |
1666 | ||
1667 | /* wait for TRCSTATR.PMSTABLE to go up */ | |
02005282 | 1668 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) { |
f188b5e7 AM |
1669 | dev_err(etm_dev, |
1670 | "timeout while waiting for PM Stable Status\n"); | |
1671 | etm4_os_unlock(drvdata); | |
1672 | ret = -EBUSY; | |
1673 | goto out; | |
1674 | } | |
1675 | ||
1676 | state = drvdata->save_state; | |
1677 | ||
f5bd5236 | 1678 | state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR); |
6288b4ce | 1679 | if (drvdata->nr_pe) |
f5bd5236 SP |
1680 | state->trcprocselr = etm4x_read32(csa, TRCPROCSELR); |
1681 | state->trcconfigr = etm4x_read32(csa, TRCCONFIGR); | |
1682 | state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR); | |
1683 | state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R); | |
1684 | state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R); | |
f7289606 SP |
1685 | if (drvdata->stallctl) |
1686 | state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR); | |
f5bd5236 SP |
1687 | state->trctsctlr = etm4x_read32(csa, TRCTSCTLR); |
1688 | state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR); | |
1689 | state->trcccctlr = etm4x_read32(csa, TRCCCCTLR); | |
1690 | state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR); | |
1691 | state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR); | |
1692 | state->trcqctlr = etm4x_read32(csa, TRCQCTLR); | |
1693 | ||
1694 | state->trcvictlr = etm4x_read32(csa, TRCVICTLR); | |
1695 | state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR); | |
1696 | state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR); | |
60c519c5 | 1697 | if (drvdata->nr_pe_cmp) |
f5bd5236 SP |
1698 | state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); |
1699 | state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR); | |
1700 | state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR); | |
1701 | state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR); | |
f188b5e7 | 1702 | |
4cd83037 | 1703 | for (i = 0; i < drvdata->nrseqstate - 1; i++) |
f5bd5236 | 1704 | state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); |
f188b5e7 | 1705 | |
589d9282 JH |
1706 | if (drvdata->nrseqstate) { |
1707 | state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR); | |
1708 | state->trcseqstr = etm4x_read32(csa, TRCSEQSTR); | |
1709 | } | |
f5bd5236 | 1710 | state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR); |
f188b5e7 AM |
1711 | |
1712 | for (i = 0; i < drvdata->nr_cntr; i++) { | |
f5bd5236 SP |
1713 | state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i)); |
1714 | state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i)); | |
1715 | state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i)); | |
f188b5e7 AM |
1716 | } |
1717 | ||
1718 | for (i = 0; i < drvdata->nr_resource * 2; i++) | |
f5bd5236 | 1719 | state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i)); |
f188b5e7 AM |
1720 | |
1721 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { | |
f5bd5236 SP |
1722 | state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i)); |
1723 | state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i)); | |
f6a18f35 | 1724 | if (etm4x_sspcicrn_present(drvdata, i)) |
f5bd5236 | 1725 | state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i)); |
f188b5e7 AM |
1726 | } |
1727 | ||
1728 | for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { | |
f5bd5236 SP |
1729 | state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i)); |
1730 | state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i)); | |
f188b5e7 AM |
1731 | } |
1732 | ||
1733 | /* | |
1734 | * Data trace stream is architecturally prohibited for A profile cores | |
1735 | * so we don't save (or later restore) trcdvcvr and trcdvcmr - As per | |
1736 | * section 1.3.4 ("Possible functional configurations of an ETMv4 trace | |
1737 | * unit") of ARM IHI 0064D. | |
1738 | */ | |
1739 | ||
1740 | for (i = 0; i < drvdata->numcidc; i++) | |
f5bd5236 | 1741 | state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i)); |
f188b5e7 AM |
1742 | |
1743 | for (i = 0; i < drvdata->numvmidc; i++) | |
f5bd5236 | 1744 | state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i)); |
f188b5e7 | 1745 | |
f5bd5236 | 1746 | state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0); |
f2603b22 | 1747 | if (drvdata->numcidc > 4) |
f5bd5236 | 1748 | state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1); |
f188b5e7 | 1749 | |
f5bd5236 | 1750 | state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0); |
93dd6440 | 1751 | if (drvdata->numvmidc > 4) |
f5bd5236 | 1752 | state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1); |
f188b5e7 | 1753 | |
f5bd5236 | 1754 | state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR); |
f188b5e7 | 1755 | |
df81b438 | 1756 | if (!drvdata->skip_power_up) |
f5bd5236 | 1757 | state->trcpdcr = etm4x_read32(csa, TRCPDCR); |
f188b5e7 AM |
1758 | |
1759 | /* wait for TRCSTATR.IDLE to go up */ | |
02005282 | 1760 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { |
f188b5e7 AM |
1761 | dev_err(etm_dev, |
1762 | "timeout while waiting for Idle Trace Status\n"); | |
1763 | etm4_os_unlock(drvdata); | |
1764 | ret = -EBUSY; | |
1765 | goto out; | |
1766 | } | |
1767 | ||
1768 | drvdata->state_needs_restore = true; | |
1769 | ||
1770 | /* | |
1771 | * Power can be removed from the trace unit now. We do this to | |
1772 | * potentially save power on systems that respect the TRCPDCR_PU | |
1773 | * despite requesting software to save/restore state. | |
1774 | */ | |
df81b438 | 1775 | if (!drvdata->skip_power_up) |
f5bd5236 SP |
1776 | etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU), |
1777 | TRCPDCR); | |
f188b5e7 | 1778 | out: |
33d5573a | 1779 | etm4_cs_lock(drvdata, csa); |
f188b5e7 AM |
1780 | return ret; |
1781 | } | |
1782 | ||
937d3f58 SP |
1783 | static int etm4_cpu_save(struct etmv4_drvdata *drvdata) |
1784 | { | |
1785 | int ret = 0; | |
1786 | ||
1787 | /* Save the TRFCR irrespective of whether the ETM is ON */ | |
5f6fd1aa | 1788 | if (drvdata->trfcr) |
937d3f58 SP |
1789 | drvdata->save_trfcr = read_trfcr(); |
1790 | /* | |
1791 | * Save and restore the ETM Trace registers only if | |
1792 | * the ETM is active. | |
1793 | */ | |
1794 | if (local_read(&drvdata->mode) && drvdata->save_state) | |
1795 | ret = __etm4_cpu_save(drvdata); | |
1796 | return ret; | |
1797 | } | |
1798 | ||
1799 | static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) | |
f188b5e7 AM |
1800 | { |
1801 | int i; | |
1802 | struct etmv4_save_state *state = drvdata->save_state; | |
f5bd5236 SP |
1803 | struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); |
1804 | struct csdev_access *csa = &tmp_csa; | |
f188b5e7 | 1805 | |
33d5573a | 1806 | etm4_cs_unlock(drvdata, csa); |
f5bd5236 | 1807 | etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); |
f188b5e7 | 1808 | |
f5bd5236 | 1809 | etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR); |
6288b4ce | 1810 | if (drvdata->nr_pe) |
f5bd5236 SP |
1811 | etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR); |
1812 | etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR); | |
1813 | etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR); | |
1814 | etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R); | |
1815 | etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R); | |
f7289606 SP |
1816 | if (drvdata->stallctl) |
1817 | etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR); | |
f5bd5236 SP |
1818 | etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR); |
1819 | etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR); | |
1820 | etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR); | |
1821 | etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR); | |
1822 | etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR); | |
1823 | etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR); | |
1824 | ||
1825 | etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR); | |
1826 | etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR); | |
1827 | etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR); | |
60c519c5 | 1828 | if (drvdata->nr_pe_cmp) |
f5bd5236 SP |
1829 | etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); |
1830 | etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR); | |
1831 | etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR); | |
1832 | etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR); | |
f188b5e7 | 1833 | |
4cd83037 | 1834 | for (i = 0; i < drvdata->nrseqstate - 1; i++) |
f5bd5236 | 1835 | etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); |
f188b5e7 | 1836 | |
589d9282 JH |
1837 | if (drvdata->nrseqstate) { |
1838 | etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR); | |
1839 | etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR); | |
1840 | } | |
f5bd5236 | 1841 | etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR); |
f188b5e7 AM |
1842 | |
1843 | for (i = 0; i < drvdata->nr_cntr; i++) { | |
f5bd5236 SP |
1844 | etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i)); |
1845 | etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i)); | |
1846 | etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i)); | |
f188b5e7 AM |
1847 | } |
1848 | ||
1849 | for (i = 0; i < drvdata->nr_resource * 2; i++) | |
f5bd5236 | 1850 | etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i)); |
f188b5e7 AM |
1851 | |
1852 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { | |
f5bd5236 SP |
1853 | etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i)); |
1854 | etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i)); | |
f6a18f35 | 1855 | if (etm4x_sspcicrn_present(drvdata, i)) |
f5bd5236 | 1856 | etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i)); |
f188b5e7 AM |
1857 | } |
1858 | ||
1859 | for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { | |
f5bd5236 SP |
1860 | etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i)); |
1861 | etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i)); | |
f188b5e7 AM |
1862 | } |
1863 | ||
1864 | for (i = 0; i < drvdata->numcidc; i++) | |
f5bd5236 | 1865 | etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i)); |
f188b5e7 AM |
1866 | |
1867 | for (i = 0; i < drvdata->numvmidc; i++) | |
f5bd5236 | 1868 | etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i)); |
f188b5e7 | 1869 | |
f5bd5236 | 1870 | etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0); |
f2603b22 | 1871 | if (drvdata->numcidc > 4) |
f5bd5236 | 1872 | etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1); |
f188b5e7 | 1873 | |
f5bd5236 | 1874 | etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0); |
93dd6440 | 1875 | if (drvdata->numvmidc > 4) |
f5bd5236 | 1876 | etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1); |
f188b5e7 | 1877 | |
f5bd5236 | 1878 | etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); |
f188b5e7 | 1879 | |
df81b438 | 1880 | if (!drvdata->skip_power_up) |
f5bd5236 | 1881 | etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR); |
f188b5e7 AM |
1882 | |
1883 | drvdata->state_needs_restore = false; | |
1884 | ||
1885 | /* | |
1886 | * As recommended by section 4.3.7 ("Synchronization when using the | |
1887 | * memory-mapped interface") of ARM IHI 0064D | |
1888 | */ | |
1889 | dsb(sy); | |
1890 | isb(); | |
1891 | ||
1892 | /* Unlock the OS lock to re-enable trace and external debug access */ | |
1893 | etm4_os_unlock(drvdata); | |
33d5573a | 1894 | etm4_cs_lock(drvdata, csa); |
f188b5e7 AM |
1895 | } |
1896 | ||
937d3f58 SP |
1897 | static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) |
1898 | { | |
5f6fd1aa | 1899 | if (drvdata->trfcr) |
937d3f58 SP |
1900 | write_trfcr(drvdata->save_trfcr); |
1901 | if (drvdata->state_needs_restore) | |
1902 | __etm4_cpu_restore(drvdata); | |
1903 | } | |
1904 | ||
f188b5e7 AM |
1905 | static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, |
1906 | void *v) | |
1907 | { | |
1908 | struct etmv4_drvdata *drvdata; | |
1909 | unsigned int cpu = smp_processor_id(); | |
1910 | ||
1911 | if (!etmdrvdata[cpu]) | |
1912 | return NOTIFY_OK; | |
1913 | ||
1914 | drvdata = etmdrvdata[cpu]; | |
1915 | ||
f188b5e7 AM |
1916 | if (WARN_ON_ONCE(drvdata->cpu != cpu)) |
1917 | return NOTIFY_BAD; | |
1918 | ||
1919 | switch (cmd) { | |
1920 | case CPU_PM_ENTER: | |
937d3f58 SP |
1921 | if (etm4_cpu_save(drvdata)) |
1922 | return NOTIFY_BAD; | |
f188b5e7 AM |
1923 | break; |
1924 | case CPU_PM_EXIT: | |
f188b5e7 | 1925 | case CPU_PM_ENTER_FAILED: |
937d3f58 | 1926 | etm4_cpu_restore(drvdata); |
f188b5e7 AM |
1927 | break; |
1928 | default: | |
1929 | return NOTIFY_DONE; | |
1930 | } | |
1931 | ||
1932 | return NOTIFY_OK; | |
1933 | } | |
1934 | ||
1935 | static struct notifier_block etm4_cpu_pm_nb = { | |
1936 | .notifier_call = etm4_cpu_pm_notify, | |
1937 | }; | |
1938 | ||
2d1a8bfb SPR |
1939 | /* Setup PM. Deals with error conditions and counts */ |
1940 | static int __init etm4_pm_setup(void) | |
f188b5e7 | 1941 | { |
9b6a3f36 | 1942 | int ret; |
500589d8 | 1943 | |
9b6a3f36 ML |
1944 | ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb); |
1945 | if (ret) | |
2d1a8bfb | 1946 | return ret; |
9b6a3f36 | 1947 | |
2d1a8bfb SPR |
1948 | ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, |
1949 | "arm/coresight4:starting", | |
1950 | etm4_starting_cpu, etm4_dying_cpu); | |
9b6a3f36 ML |
1951 | |
1952 | if (ret) | |
1953 | goto unregister_notifier; | |
1954 | ||
2d1a8bfb SPR |
1955 | ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, |
1956 | "arm/coresight4:online", | |
1957 | etm4_online_cpu, NULL); | |
9b6a3f36 ML |
1958 | |
1959 | /* HP dyn state ID returned in ret on success */ | |
1960 | if (ret > 0) { | |
1961 | hp_online = ret; | |
1962 | return 0; | |
1963 | } | |
1964 | ||
1965 | /* failed dyn state - remove others */ | |
2d1a8bfb | 1966 | cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); |
9b6a3f36 ML |
1967 | |
1968 | unregister_notifier: | |
1969 | cpu_pm_unregister_notifier(&etm4_cpu_pm_nb); | |
9b6a3f36 | 1970 | return ret; |
f188b5e7 AM |
1971 | } |
1972 | ||
22a550a3 | 1973 | static void etm4_pm_clear(void) |
f188b5e7 | 1974 | { |
9b6a3f36 ML |
1975 | cpu_pm_unregister_notifier(&etm4_cpu_pm_nb); |
1976 | cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); | |
1977 | if (hp_online) { | |
1978 | cpuhp_remove_state_nocalls(hp_online); | |
1979 | hp_online = 0; | |
1980 | } | |
f188b5e7 | 1981 | } |
f188b5e7 | 1982 | |
3c728e07 | 1983 | static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) |
2e1cdfe1 PP |
1984 | { |
1985 | int ret; | |
2e1cdfe1 | 1986 | struct coresight_platform_data *pdata = NULL; |
3c728e07 TZ |
1987 | struct device *dev = init_arg->dev; |
1988 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); | |
9486295a | 1989 | struct coresight_desc desc = { 0 }; |
35e1c916 SP |
1990 | u8 major, minor; |
1991 | char *type_name; | |
2e1cdfe1 | 1992 | |
2e1cdfe1 | 1993 | if (!drvdata) |
3c728e07 | 1994 | return -EINVAL; |
fd6e7905 | 1995 | |
3c728e07 | 1996 | desc.access = *init_arg->csa; |
2e1cdfe1 | 1997 | |
8b94db1e | 1998 | if (!drvdata->arch) |
22a550a3 | 1999 | return -EINVAL; |
fc208abe | 2000 | |
5214b563 SP |
2001 | /* TRCPDCR is not accessible with system instructions. */ |
2002 | if (!desc.access.io_mem || | |
2003 | fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) | |
2004 | drvdata->skip_power_up = true; | |
2005 | ||
35e1c916 SP |
2006 | major = ETM_ARCH_MAJOR_VERSION(drvdata->arch); |
2007 | minor = ETM_ARCH_MINOR_VERSION(drvdata->arch); | |
2008 | ||
2009 | if (etm4x_is_ete(drvdata)) { | |
2010 | type_name = "ete"; | |
2011 | /* ETE v1 has major version == 0b101. Adjust this for logging.*/ | |
2012 | major -= 4; | |
2013 | } else { | |
2014 | type_name = "etm"; | |
2015 | } | |
2016 | ||
2017 | desc.name = devm_kasprintf(dev, GFP_KERNEL, | |
2018 | "%s%d", type_name, drvdata->cpu); | |
2019 | if (!desc.name) | |
2020 | return -ENOMEM; | |
2021 | ||
fc208abe | 2022 | etm4_set_default(&drvdata->config); |
2e1cdfe1 | 2023 | |
af7cfd0f | 2024 | pdata = coresight_get_platform_data(dev); |
22a550a3 KP |
2025 | if (IS_ERR(pdata)) |
2026 | return PTR_ERR(pdata); | |
2027 | ||
c23bc382 | 2028 | dev->platform_data = pdata; |
af7cfd0f | 2029 | |
9486295a SP |
2030 | desc.type = CORESIGHT_DEV_TYPE_SOURCE; |
2031 | desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; | |
2032 | desc.ops = &etm4_cs_ops; | |
2033 | desc.pdata = pdata; | |
2034 | desc.dev = dev; | |
2035 | desc.groups = coresight_etmv4_groups; | |
2036 | drvdata->csdev = coresight_register(&desc); | |
22a550a3 KP |
2037 | if (IS_ERR(drvdata->csdev)) |
2038 | return PTR_ERR(drvdata->csdev); | |
2e1cdfe1 | 2039 | |
37fbbdbd MP |
2040 | ret = etm_perf_symlink(drvdata->csdev, true); |
2041 | if (ret) { | |
2042 | coresight_unregister(drvdata->csdev); | |
22a550a3 | 2043 | return ret; |
37fbbdbd MP |
2044 | } |
2045 | ||
810ac401 ML |
2046 | /* register with config infrastructure & load any current features */ |
2047 | ret = etm4_cscfg_register(drvdata->csdev); | |
2048 | if (ret) { | |
2049 | coresight_unregister(drvdata->csdev); | |
2050 | return ret; | |
2051 | } | |
2052 | ||
22a550a3 KP |
2053 | etmdrvdata[drvdata->cpu] = drvdata; |
2054 | ||
35e1c916 SP |
2055 | dev_info(&drvdata->csdev->dev, "CPU%d: %s v%d.%d initialized\n", |
2056 | drvdata->cpu, type_name, major, minor); | |
2e1cdfe1 PP |
2057 | |
2058 | if (boot_enable) { | |
2059 | coresight_enable(drvdata->csdev); | |
2060 | drvdata->boot_enable = true; | |
2061 | } | |
2062 | ||
2063 | return 0; | |
2e1cdfe1 PP |
2064 | } |
2065 | ||
5a1c7097 | 2066 | static int etm4_probe(struct device *dev) |
3c728e07 | 2067 | { |
3095e90e | 2068 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); |
3c728e07 TZ |
2069 | struct csdev_access access = { 0 }; |
2070 | struct etm4_init_arg init_arg = { 0 }; | |
2071 | struct etm4_init_arg *delayed; | |
2072 | ||
3095e90e | 2073 | if (WARN_ON(!drvdata)) |
3c728e07 TZ |
2074 | return -ENOMEM; |
2075 | ||
3c728e07 TZ |
2076 | if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE) |
2077 | pm_save_enable = coresight_loses_context_with_cpu(dev) ? | |
2078 | PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER; | |
2079 | ||
2080 | if (pm_save_enable != PARAM_PM_SAVE_NEVER) { | |
2081 | drvdata->save_state = devm_kmalloc(dev, | |
2082 | sizeof(struct etmv4_save_state), GFP_KERNEL); | |
2083 | if (!drvdata->save_state) | |
2084 | return -ENOMEM; | |
2085 | } | |
2086 | ||
3c728e07 TZ |
2087 | spin_lock_init(&drvdata->spinlock); |
2088 | ||
2089 | drvdata->cpu = coresight_get_cpu(dev); | |
2090 | if (drvdata->cpu < 0) | |
2091 | return drvdata->cpu; | |
2092 | ||
2093 | init_arg.dev = dev; | |
2094 | init_arg.csa = &access; | |
3c728e07 TZ |
2095 | |
2096 | /* | |
2097 | * Serialize against CPUHP callbacks to avoid race condition | |
2098 | * between the smp call and saving the delayed probe. | |
2099 | */ | |
2100 | cpus_read_lock(); | |
2101 | if (smp_call_function_single(drvdata->cpu, | |
2102 | etm4_init_arch_data, &init_arg, 1)) { | |
2103 | /* The CPU was offline, try again once it comes online. */ | |
2104 | delayed = devm_kmalloc(dev, sizeof(*delayed), GFP_KERNEL); | |
2105 | if (!delayed) { | |
2106 | cpus_read_unlock(); | |
2107 | return -ENOMEM; | |
2108 | } | |
2109 | ||
2110 | *delayed = init_arg; | |
2111 | ||
2112 | per_cpu(delayed_probe, drvdata->cpu) = delayed; | |
2113 | ||
2114 | cpus_read_unlock(); | |
2115 | return 0; | |
2116 | } | |
2117 | cpus_read_unlock(); | |
2118 | ||
2119 | return etm4_add_coresight_dev(&init_arg); | |
2120 | } | |
2121 | ||
c23bc382 SP |
2122 | static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id) |
2123 | { | |
3095e90e | 2124 | struct etmv4_drvdata *drvdata; |
c23bc382 SP |
2125 | void __iomem *base; |
2126 | struct device *dev = &adev->dev; | |
2127 | struct resource *res = &adev->res; | |
2128 | int ret; | |
2129 | ||
2130 | /* Validity for the resource is already checked by the AMBA core */ | |
2131 | base = devm_ioremap_resource(dev, res); | |
2132 | if (IS_ERR(base)) | |
2133 | return PTR_ERR(base); | |
2134 | ||
3095e90e AK |
2135 | drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); |
2136 | if (!drvdata) | |
2137 | return -ENOMEM; | |
2138 | ||
4e3b9a6e | 2139 | drvdata->base = base; |
3095e90e | 2140 | dev_set_drvdata(dev, drvdata); |
5a1c7097 | 2141 | ret = etm4_probe(dev); |
c23bc382 SP |
2142 | if (!ret) |
2143 | pm_runtime_put(&adev->dev); | |
2144 | ||
2145 | return ret; | |
2146 | } | |
2147 | ||
5214b563 SP |
2148 | static int etm4_probe_platform_dev(struct platform_device *pdev) |
2149 | { | |
73d779a0 | 2150 | struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
3095e90e | 2151 | struct etmv4_drvdata *drvdata; |
5214b563 SP |
2152 | int ret; |
2153 | ||
3095e90e AK |
2154 | drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL); |
2155 | if (!drvdata) | |
2156 | return -ENOMEM; | |
2157 | ||
73d779a0 AK |
2158 | drvdata->pclk = coresight_get_enable_apb_pclk(&pdev->dev); |
2159 | if (IS_ERR(drvdata->pclk)) | |
2160 | return -ENODEV; | |
2161 | ||
2162 | if (res) { | |
2163 | drvdata->base = devm_ioremap_resource(&pdev->dev, res); | |
2164 | if (IS_ERR(drvdata->base)) { | |
2165 | clk_put(drvdata->pclk); | |
2166 | return PTR_ERR(drvdata->base); | |
2167 | } | |
2168 | } | |
2169 | ||
3095e90e | 2170 | dev_set_drvdata(&pdev->dev, drvdata); |
5214b563 SP |
2171 | pm_runtime_get_noresume(&pdev->dev); |
2172 | pm_runtime_set_active(&pdev->dev); | |
2173 | pm_runtime_enable(&pdev->dev); | |
2174 | ||
5a1c7097 | 2175 | ret = etm4_probe(&pdev->dev); |
5214b563 SP |
2176 | |
2177 | pm_runtime_put(&pdev->dev); | |
2178 | return ret; | |
2179 | } | |
2180 | ||
3c728e07 TZ |
2181 | static int etm4_probe_cpu(unsigned int cpu) |
2182 | { | |
2183 | int ret; | |
2184 | struct etm4_init_arg init_arg; | |
2185 | struct csdev_access access = { 0 }; | |
2186 | struct etm4_init_arg *iap = *this_cpu_ptr(&delayed_probe); | |
2187 | ||
2188 | if (!iap) | |
2189 | return 0; | |
2190 | ||
2191 | init_arg = *iap; | |
2192 | devm_kfree(init_arg.dev, iap); | |
2193 | *this_cpu_ptr(&delayed_probe) = NULL; | |
2194 | ||
2195 | ret = pm_runtime_resume_and_get(init_arg.dev); | |
2196 | if (ret < 0) { | |
2197 | dev_err(init_arg.dev, "Failed to get PM runtime!\n"); | |
2198 | return 0; | |
2199 | } | |
2200 | ||
2201 | init_arg.csa = &access; | |
2202 | etm4_init_arch_data(&init_arg); | |
2203 | ||
2204 | etm4_add_coresight_dev(&init_arg); | |
2205 | ||
2206 | pm_runtime_put(init_arg.dev); | |
2207 | return 0; | |
2208 | } | |
2209 | ||
28941701 ML |
2210 | static struct amba_cs_uci_id uci_id_etm4[] = { |
2211 | { | |
2212 | /* ETMv4 UCI data */ | |
d02dfac3 SP |
2213 | .devarch = ETM_DEVARCH_ETMv4x_ARCH, |
2214 | .devarch_mask = ETM_DEVARCH_ID_MASK, | |
73d779a0 | 2215 | .devtype = CS_DEVTYPE_PE_TRACE, |
5cedd223 | 2216 | } |
28941701 | 2217 | }; |
5cedd223 | 2218 | |
45fe7bef | 2219 | static void clear_etmdrvdata(void *info) |
22a550a3 KP |
2220 | { |
2221 | int cpu = *(int *)info; | |
2222 | ||
2223 | etmdrvdata[cpu] = NULL; | |
3c728e07 | 2224 | per_cpu(delayed_probe, cpu) = NULL; |
22a550a3 KP |
2225 | } |
2226 | ||
348ddab8 | 2227 | static void etm4_remove_dev(struct etmv4_drvdata *drvdata) |
22a550a3 | 2228 | { |
3c728e07 | 2229 | bool had_delayed_probe; |
22a550a3 | 2230 | /* |
c23bc382 SP |
2231 | * Taking hotplug lock here to avoid racing between etm4_remove_dev() |
2232 | * and CPU hotplug call backs. | |
22a550a3 KP |
2233 | */ |
2234 | cpus_read_lock(); | |
3c728e07 TZ |
2235 | |
2236 | had_delayed_probe = per_cpu(delayed_probe, drvdata->cpu); | |
2237 | ||
22a550a3 KP |
2238 | /* |
2239 | * The readers for etmdrvdata[] are CPU hotplug call backs | |
2240 | * and PM notification call backs. Change etmdrvdata[i] on | |
2241 | * CPU i ensures these call backs has consistent view | |
2242 | * inside one call back function. | |
2243 | */ | |
2244 | if (smp_call_function_single(drvdata->cpu, clear_etmdrvdata, &drvdata->cpu, 1)) | |
3c728e07 | 2245 | clear_etmdrvdata(&drvdata->cpu); |
22a550a3 KP |
2246 | |
2247 | cpus_read_unlock(); | |
2248 | ||
3c728e07 TZ |
2249 | if (!had_delayed_probe) { |
2250 | etm_perf_symlink(drvdata->csdev, false); | |
2251 | cscfg_unregister_csdev(drvdata->csdev); | |
2252 | coresight_unregister(drvdata->csdev); | |
2253 | } | |
22a550a3 KP |
2254 | } |
2255 | ||
348ddab8 | 2256 | static void etm4_remove_amba(struct amba_device *adev) |
c23bc382 SP |
2257 | { |
2258 | struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); | |
2259 | ||
2260 | if (drvdata) | |
1609faa9 | 2261 | etm4_remove_dev(drvdata); |
c23bc382 SP |
2262 | } |
2263 | ||
348ddab8 | 2264 | static int etm4_remove_platform_dev(struct platform_device *pdev) |
5214b563 | 2265 | { |
5214b563 SP |
2266 | struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev); |
2267 | ||
2268 | if (drvdata) | |
c5f231f1 | 2269 | etm4_remove_dev(drvdata); |
5214b563 | 2270 | pm_runtime_disable(&pdev->dev); |
73d779a0 | 2271 | |
a4621fd1 | 2272 | if (drvdata && !IS_ERR_OR_NULL(drvdata->pclk)) |
73d779a0 AK |
2273 | clk_put(drvdata->pclk); |
2274 | ||
c5f231f1 | 2275 | return 0; |
5214b563 SP |
2276 | } |
2277 | ||
c5520c93 | 2278 | static const struct amba_id etm4_ids[] = { |
17b4add0 SPR |
2279 | CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */ |
2280 | CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */ | |
2281 | CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */ | |
2282 | CS_AMBA_ID(0x000bb959), /* Cortex-A73 */ | |
2283 | CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */ | |
b8336ad9 CZ |
2284 | CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),/* Cortex-A55 */ |
2285 | CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),/* Cortex-A75 */ | |
fac28c4d | 2286 | CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */ |
ec585949 | 2287 | CS_AMBA_UCI_ID(0x000bbd41, uci_id_etm4),/* Cortex-A78 */ |
17b4add0 SPR |
2288 | CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */ |
2289 | CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */ | |
41e8c720 SPR |
2290 | CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */ |
2291 | CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */ | |
63314ca2 SPR |
2292 | CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */ |
2293 | CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */ | |
0605b89d | 2294 | CS_AMBA_UCI_ID(0x000bbd0d, uci_id_etm4),/* Qualcomm Kryo 5XX Cortex-A77 */ |
0373d906 | 2295 | CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */ |
447a612e QL |
2296 | CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),/* HiSilicon-Hip08 */ |
2297 | CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),/* HiSilicon-Hip09 */ | |
ab5ca626 SP |
2298 | /* |
2299 | * Match all PIDs with ETM4 DEVARCH. No need for adding any of the new | |
2300 | * CPUs to the list here. | |
2301 | */ | |
2302 | CS_AMBA_MATCH_ALL_UCI(uci_id_etm4), | |
5cedd223 | 2303 | {}, |
2e1cdfe1 PP |
2304 | }; |
2305 | ||
22a550a3 KP |
2306 | MODULE_DEVICE_TABLE(amba, etm4_ids); |
2307 | ||
c23bc382 | 2308 | static struct amba_driver etm4x_amba_driver = { |
2e1cdfe1 PP |
2309 | .drv = { |
2310 | .name = "coresight-etm4x", | |
22a550a3 | 2311 | .owner = THIS_MODULE, |
b15f0fb6 | 2312 | .suppress_bind_attrs = true, |
2e1cdfe1 | 2313 | }, |
c23bc382 SP |
2314 | .probe = etm4_probe_amba, |
2315 | .remove = etm4_remove_amba, | |
2e1cdfe1 PP |
2316 | .id_table = etm4_ids, |
2317 | }; | |
2d1a8bfb | 2318 | |
73d779a0 AK |
2319 | #ifdef CONFIG_PM |
2320 | static int etm4_runtime_suspend(struct device *dev) | |
2321 | { | |
2322 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); | |
2323 | ||
2324 | if (drvdata->pclk && !IS_ERR(drvdata->pclk)) | |
2325 | clk_disable_unprepare(drvdata->pclk); | |
2326 | ||
2327 | return 0; | |
2328 | } | |
2329 | ||
2330 | static int etm4_runtime_resume(struct device *dev) | |
2331 | { | |
2332 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); | |
2333 | ||
2334 | if (drvdata->pclk && !IS_ERR(drvdata->pclk)) | |
2335 | clk_prepare_enable(drvdata->pclk); | |
2336 | ||
2337 | return 0; | |
2338 | } | |
2339 | #endif | |
2340 | ||
2341 | static const struct dev_pm_ops etm4_dev_pm_ops = { | |
2342 | SET_RUNTIME_PM_OPS(etm4_runtime_suspend, etm4_runtime_resume, NULL) | |
2343 | }; | |
2344 | ||
5214b563 SP |
2345 | static const struct of_device_id etm4_sysreg_match[] = { |
2346 | { .compatible = "arm,coresight-etm4x-sysreg" }, | |
35e1c916 | 2347 | { .compatible = "arm,embedded-trace-extension" }, |
5214b563 SP |
2348 | {} |
2349 | }; | |
2350 | ||
134124ac SP |
2351 | #ifdef CONFIG_ACPI |
2352 | static const struct acpi_device_id etm4x_acpi_ids[] = { | |
2353 | {"ARMHC500", 0}, /* ARM CoreSight ETM4x */ | |
2354 | {} | |
2355 | }; | |
2356 | MODULE_DEVICE_TABLE(acpi, etm4x_acpi_ids); | |
2357 | #endif | |
2358 | ||
5214b563 SP |
2359 | static struct platform_driver etm4_platform_driver = { |
2360 | .probe = etm4_probe_platform_dev, | |
2361 | .remove = etm4_remove_platform_dev, | |
2362 | .driver = { | |
2363 | .name = "coresight-etm4x", | |
2364 | .of_match_table = etm4_sysreg_match, | |
134124ac | 2365 | .acpi_match_table = ACPI_PTR(etm4x_acpi_ids), |
5214b563 | 2366 | .suppress_bind_attrs = true, |
73d779a0 | 2367 | .pm = &etm4_dev_pm_ops, |
5214b563 SP |
2368 | }, |
2369 | }; | |
2370 | ||
2d1a8bfb SPR |
2371 | static int __init etm4x_init(void) |
2372 | { | |
2373 | int ret; | |
2374 | ||
2375 | ret = etm4_pm_setup(); | |
2376 | ||
2377 | /* etm4_pm_setup() does its own cleanup - exit on error */ | |
2378 | if (ret) | |
2379 | return ret; | |
2380 | ||
c23bc382 | 2381 | ret = amba_driver_register(&etm4x_amba_driver); |
2d1a8bfb | 2382 | if (ret) { |
5214b563 SP |
2383 | pr_err("Error registering etm4x AMBA driver\n"); |
2384 | goto clear_pm; | |
2d1a8bfb SPR |
2385 | } |
2386 | ||
5214b563 SP |
2387 | ret = platform_driver_register(&etm4_platform_driver); |
2388 | if (!ret) | |
2389 | return 0; | |
2390 | ||
2391 | pr_err("Error registering etm4x platform driver\n"); | |
2392 | amba_driver_unregister(&etm4x_amba_driver); | |
2393 | ||
2394 | clear_pm: | |
2395 | etm4_pm_clear(); | |
2d1a8bfb SPR |
2396 | return ret; |
2397 | } | |
22a550a3 KP |
2398 | |
2399 | static void __exit etm4x_exit(void) | |
2400 | { | |
c23bc382 | 2401 | amba_driver_unregister(&etm4x_amba_driver); |
5214b563 | 2402 | platform_driver_unregister(&etm4_platform_driver); |
22a550a3 KP |
2403 | etm4_pm_clear(); |
2404 | } | |
2405 | ||
2406 | module_init(etm4x_init); | |
2407 | module_exit(etm4x_exit); | |
2408 | ||
2409 | MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>"); | |
2410 | MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>"); | |
2411 | MODULE_DESCRIPTION("Arm CoreSight Program Flow Trace v4.x driver"); | |
2412 | MODULE_LICENSE("GPL v2"); |