]>
Commit | Line | Data |
---|---|---|
178e9fc4 AK |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* | |
3 | * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> | |
4 | * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar | |
5 | * Copyright (C) 2009 Jaswinder Singh Rajput | |
6 | * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter | |
7 | * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra | |
8 | * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> | |
9 | * Copyright (C) 2009 Google, Inc., Stephane Eranian | |
10 | * Copyright 2014 Tilera Corporation. All Rights Reserved. | |
11 | * Copyright (C) 2018 Andes Technology Corporation | |
12 | * | |
13 | * Perf_events support for RISC-V platforms. | |
14 | * | |
15 | * Since the spec. (as of now, Priv-Spec 1.10) does not provide enough | |
16 | * functionality for perf event to fully work, this file provides | |
17 | * the very basic framework only. | |
18 | * | |
19 | * For platform portings, please check Documentations/riscv/pmu.txt. | |
20 | * | |
21 | * The Copyright line includes x86 and tile ones. | |
22 | */ | |
23 | ||
24 | #include <linux/kprobes.h> | |
25 | #include <linux/kernel.h> | |
26 | #include <linux/kdebug.h> | |
27 | #include <linux/mutex.h> | |
28 | #include <linux/bitmap.h> | |
29 | #include <linux/irq.h> | |
178e9fc4 AK |
30 | #include <linux/perf_event.h> |
31 | #include <linux/atomic.h> | |
32 | #include <linux/of.h> | |
33 | #include <asm/perf_event.h> | |
34 | ||
35 | static const struct riscv_pmu *riscv_pmu __read_mostly; | |
36 | static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); | |
37 | ||
38 | /* | |
39 | * Hardware & cache maps and their methods | |
40 | */ | |
41 | ||
42 | static const int riscv_hw_event_map[] = { | |
43 | [PERF_COUNT_HW_CPU_CYCLES] = RISCV_PMU_CYCLE, | |
44 | [PERF_COUNT_HW_INSTRUCTIONS] = RISCV_PMU_INSTRET, | |
45 | [PERF_COUNT_HW_CACHE_REFERENCES] = RISCV_OP_UNSUPP, | |
46 | [PERF_COUNT_HW_CACHE_MISSES] = RISCV_OP_UNSUPP, | |
47 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = RISCV_OP_UNSUPP, | |
48 | [PERF_COUNT_HW_BRANCH_MISSES] = RISCV_OP_UNSUPP, | |
49 | [PERF_COUNT_HW_BUS_CYCLES] = RISCV_OP_UNSUPP, | |
50 | }; | |
51 | ||
52 | #define C(x) PERF_COUNT_HW_CACHE_##x | |
53 | static const int riscv_cache_event_map[PERF_COUNT_HW_CACHE_MAX] | |
54 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
55 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | |
56 | [C(L1D)] = { | |
57 | [C(OP_READ)] = { | |
58 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
59 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
60 | }, | |
61 | [C(OP_WRITE)] = { | |
62 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
63 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
64 | }, | |
65 | [C(OP_PREFETCH)] = { | |
66 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
67 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
68 | }, | |
69 | }, | |
70 | [C(L1I)] = { | |
71 | [C(OP_READ)] = { | |
72 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
73 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
74 | }, | |
75 | [C(OP_WRITE)] = { | |
76 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
77 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
78 | }, | |
79 | [C(OP_PREFETCH)] = { | |
80 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
81 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
82 | }, | |
83 | }, | |
84 | [C(LL)] = { | |
85 | [C(OP_READ)] = { | |
86 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
87 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
88 | }, | |
89 | [C(OP_WRITE)] = { | |
90 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
91 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
92 | }, | |
93 | [C(OP_PREFETCH)] = { | |
94 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
95 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
96 | }, | |
97 | }, | |
98 | [C(DTLB)] = { | |
99 | [C(OP_READ)] = { | |
100 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
101 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
102 | }, | |
103 | [C(OP_WRITE)] = { | |
104 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
105 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
106 | }, | |
107 | [C(OP_PREFETCH)] = { | |
108 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
109 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
110 | }, | |
111 | }, | |
112 | [C(ITLB)] = { | |
113 | [C(OP_READ)] = { | |
114 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
115 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
116 | }, | |
117 | [C(OP_WRITE)] = { | |
118 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
119 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
120 | }, | |
121 | [C(OP_PREFETCH)] = { | |
122 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
123 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
124 | }, | |
125 | }, | |
126 | [C(BPU)] = { | |
127 | [C(OP_READ)] = { | |
128 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
129 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
130 | }, | |
131 | [C(OP_WRITE)] = { | |
132 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
133 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
134 | }, | |
135 | [C(OP_PREFETCH)] = { | |
136 | [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP, | |
137 | [C(RESULT_MISS)] = RISCV_OP_UNSUPP, | |
138 | }, | |
139 | }, | |
140 | }; | |
141 | ||
142 | static int riscv_map_hw_event(u64 config) | |
143 | { | |
144 | if (config >= riscv_pmu->max_events) | |
145 | return -EINVAL; | |
146 | ||
147 | return riscv_pmu->hw_events[config]; | |
148 | } | |
149 | ||
e7b146a8 | 150 | static int riscv_map_cache_decode(u64 config, unsigned int *type, |
178e9fc4 AK |
151 | unsigned int *op, unsigned int *result) |
152 | { | |
153 | return -ENOENT; | |
154 | } | |
155 | ||
156 | static int riscv_map_cache_event(u64 config) | |
157 | { | |
158 | unsigned int type, op, result; | |
159 | int err = -ENOENT; | |
160 | int code; | |
161 | ||
162 | err = riscv_map_cache_decode(config, &type, &op, &result); | |
163 | if (!riscv_pmu->cache_events || err) | |
164 | return err; | |
165 | ||
166 | if (type >= PERF_COUNT_HW_CACHE_MAX || | |
167 | op >= PERF_COUNT_HW_CACHE_OP_MAX || | |
168 | result >= PERF_COUNT_HW_CACHE_RESULT_MAX) | |
169 | return -EINVAL; | |
170 | ||
171 | code = (*riscv_pmu->cache_events)[type][op][result]; | |
172 | if (code == RISCV_OP_UNSUPP) | |
173 | return -EINVAL; | |
174 | ||
175 | return code; | |
176 | } | |
177 | ||
178 | /* | |
179 | * Low-level functions: reading/writing counters | |
180 | */ | |
181 | ||
182 | static inline u64 read_counter(int idx) | |
183 | { | |
184 | u64 val = 0; | |
185 | ||
186 | switch (idx) { | |
187 | case RISCV_PMU_CYCLE: | |
a3182c91 | 188 | val = csr_read(CSR_CYCLE); |
178e9fc4 AK |
189 | break; |
190 | case RISCV_PMU_INSTRET: | |
a3182c91 | 191 | val = csr_read(CSR_INSTRET); |
178e9fc4 AK |
192 | break; |
193 | default: | |
194 | WARN_ON_ONCE(idx < 0 || idx > RISCV_MAX_COUNTERS); | |
195 | return -EINVAL; | |
196 | } | |
197 | ||
198 | return val; | |
199 | } | |
200 | ||
201 | static inline void write_counter(int idx, u64 value) | |
202 | { | |
203 | /* currently not supported */ | |
204 | WARN_ON_ONCE(1); | |
205 | } | |
206 | ||
207 | /* | |
208 | * pmu->read: read and update the counter | |
209 | * | |
210 | * Other architectures' implementation often have a xxx_perf_event_update | |
211 | * routine, which can return counter values when called in the IRQ, but | |
212 | * return void when being called by the pmu->read method. | |
213 | */ | |
214 | static void riscv_pmu_read(struct perf_event *event) | |
215 | { | |
216 | struct hw_perf_event *hwc = &event->hw; | |
217 | u64 prev_raw_count, new_raw_count; | |
218 | u64 oldval; | |
219 | int idx = hwc->idx; | |
220 | u64 delta; | |
221 | ||
222 | do { | |
223 | prev_raw_count = local64_read(&hwc->prev_count); | |
224 | new_raw_count = read_counter(idx); | |
225 | ||
226 | oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count, | |
227 | new_raw_count); | |
228 | } while (oldval != prev_raw_count); | |
229 | ||
230 | /* | |
231 | * delta is the value to update the counter we maintain in the kernel. | |
232 | */ | |
233 | delta = (new_raw_count - prev_raw_count) & | |
234 | ((1ULL << riscv_pmu->counter_width) - 1); | |
235 | local64_add(delta, &event->count); | |
236 | /* | |
237 | * Something like local64_sub(delta, &hwc->period_left) here is | |
238 | * needed if there is an interrupt for perf. | |
239 | */ | |
240 | } | |
241 | ||
242 | /* | |
243 | * State transition functions: | |
244 | * | |
245 | * stop()/start() & add()/del() | |
246 | */ | |
247 | ||
248 | /* | |
249 | * pmu->stop: stop the counter | |
250 | */ | |
251 | static void riscv_pmu_stop(struct perf_event *event, int flags) | |
252 | { | |
253 | struct hw_perf_event *hwc = &event->hw; | |
254 | ||
255 | WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); | |
256 | hwc->state |= PERF_HES_STOPPED; | |
257 | ||
258 | if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { | |
259 | riscv_pmu->pmu->read(event); | |
260 | hwc->state |= PERF_HES_UPTODATE; | |
261 | } | |
262 | } | |
263 | ||
264 | /* | |
265 | * pmu->start: start the event. | |
266 | */ | |
267 | static void riscv_pmu_start(struct perf_event *event, int flags) | |
268 | { | |
269 | struct hw_perf_event *hwc = &event->hw; | |
270 | ||
271 | if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) | |
272 | return; | |
273 | ||
274 | if (flags & PERF_EF_RELOAD) { | |
275 | WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); | |
276 | ||
277 | /* | |
278 | * Set the counter to the period to the next interrupt here, | |
279 | * if you have any. | |
280 | */ | |
281 | } | |
282 | ||
283 | hwc->state = 0; | |
284 | perf_event_update_userpage(event); | |
285 | ||
286 | /* | |
287 | * Since we cannot write to counters, this serves as an initialization | |
288 | * to the delta-mechanism in pmu->read(); otherwise, the delta would be | |
289 | * wrong when pmu->read is called for the first time. | |
290 | */ | |
291 | local64_set(&hwc->prev_count, read_counter(hwc->idx)); | |
292 | } | |
293 | ||
294 | /* | |
295 | * pmu->add: add the event to PMU. | |
296 | */ | |
297 | static int riscv_pmu_add(struct perf_event *event, int flags) | |
298 | { | |
299 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); | |
300 | struct hw_perf_event *hwc = &event->hw; | |
301 | ||
302 | if (cpuc->n_events == riscv_pmu->num_counters) | |
303 | return -ENOSPC; | |
304 | ||
305 | /* | |
306 | * We don't have general conunters, so no binding-event-to-counter | |
307 | * process here. | |
308 | * | |
309 | * Indexing using hwc->config generally not works, since config may | |
310 | * contain extra information, but here the only info we have in | |
311 | * hwc->config is the event index. | |
312 | */ | |
313 | hwc->idx = hwc->config; | |
314 | cpuc->events[hwc->idx] = event; | |
315 | cpuc->n_events++; | |
316 | ||
317 | hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; | |
318 | ||
319 | if (flags & PERF_EF_START) | |
320 | riscv_pmu->pmu->start(event, PERF_EF_RELOAD); | |
321 | ||
322 | return 0; | |
323 | } | |
324 | ||
325 | /* | |
326 | * pmu->del: delete the event from PMU. | |
327 | */ | |
328 | static void riscv_pmu_del(struct perf_event *event, int flags) | |
329 | { | |
330 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); | |
331 | struct hw_perf_event *hwc = &event->hw; | |
332 | ||
333 | cpuc->events[hwc->idx] = NULL; | |
334 | cpuc->n_events--; | |
335 | riscv_pmu->pmu->stop(event, PERF_EF_UPDATE); | |
336 | perf_event_update_userpage(event); | |
337 | } | |
338 | ||
339 | /* | |
340 | * Interrupt: a skeletion for reference. | |
341 | */ | |
342 | ||
343 | static DEFINE_MUTEX(pmc_reserve_mutex); | |
344 | ||
e7b146a8 | 345 | static irqreturn_t riscv_base_pmu_handle_irq(int irq_num, void *dev) |
178e9fc4 AK |
346 | { |
347 | return IRQ_NONE; | |
348 | } | |
349 | ||
350 | static int reserve_pmc_hardware(void) | |
351 | { | |
352 | int err = 0; | |
353 | ||
354 | mutex_lock(&pmc_reserve_mutex); | |
355 | if (riscv_pmu->irq >= 0 && riscv_pmu->handle_irq) { | |
356 | err = request_irq(riscv_pmu->irq, riscv_pmu->handle_irq, | |
357 | IRQF_PERCPU, "riscv-base-perf", NULL); | |
358 | } | |
359 | mutex_unlock(&pmc_reserve_mutex); | |
360 | ||
361 | return err; | |
362 | } | |
363 | ||
e7b146a8 | 364 | static void release_pmc_hardware(void) |
178e9fc4 AK |
365 | { |
366 | mutex_lock(&pmc_reserve_mutex); | |
367 | if (riscv_pmu->irq >= 0) | |
368 | free_irq(riscv_pmu->irq, NULL); | |
369 | mutex_unlock(&pmc_reserve_mutex); | |
370 | } | |
371 | ||
372 | /* | |
373 | * Event Initialization/Finalization | |
374 | */ | |
375 | ||
376 | static atomic_t riscv_active_events = ATOMIC_INIT(0); | |
377 | ||
378 | static void riscv_event_destroy(struct perf_event *event) | |
379 | { | |
380 | if (atomic_dec_return(&riscv_active_events) == 0) | |
381 | release_pmc_hardware(); | |
382 | } | |
383 | ||
384 | static int riscv_event_init(struct perf_event *event) | |
385 | { | |
386 | struct perf_event_attr *attr = &event->attr; | |
387 | struct hw_perf_event *hwc = &event->hw; | |
388 | int err; | |
389 | int code; | |
390 | ||
391 | if (atomic_inc_return(&riscv_active_events) == 1) { | |
392 | err = reserve_pmc_hardware(); | |
393 | ||
394 | if (err) { | |
395 | pr_warn("PMC hardware not available\n"); | |
396 | atomic_dec(&riscv_active_events); | |
397 | return -EBUSY; | |
398 | } | |
399 | } | |
400 | ||
401 | switch (event->attr.type) { | |
402 | case PERF_TYPE_HARDWARE: | |
403 | code = riscv_pmu->map_hw_event(attr->config); | |
404 | break; | |
405 | case PERF_TYPE_HW_CACHE: | |
406 | code = riscv_pmu->map_cache_event(attr->config); | |
407 | break; | |
408 | case PERF_TYPE_RAW: | |
409 | return -EOPNOTSUPP; | |
410 | default: | |
411 | return -ENOENT; | |
412 | } | |
413 | ||
414 | event->destroy = riscv_event_destroy; | |
415 | if (code < 0) { | |
416 | event->destroy(event); | |
417 | return code; | |
418 | } | |
419 | ||
420 | /* | |
421 | * idx is set to -1 because the index of a general event should not be | |
422 | * decided until binding to some counter in pmu->add(). | |
423 | * | |
424 | * But since we don't have such support, later in pmu->add(), we just | |
425 | * use hwc->config as the index instead. | |
426 | */ | |
427 | hwc->config = code; | |
428 | hwc->idx = -1; | |
429 | ||
430 | return 0; | |
431 | } | |
432 | ||
433 | /* | |
434 | * Initialization | |
435 | */ | |
436 | ||
437 | static struct pmu min_pmu = { | |
438 | .name = "riscv-base", | |
439 | .event_init = riscv_event_init, | |
440 | .add = riscv_pmu_add, | |
441 | .del = riscv_pmu_del, | |
442 | .start = riscv_pmu_start, | |
443 | .stop = riscv_pmu_stop, | |
444 | .read = riscv_pmu_read, | |
445 | }; | |
446 | ||
447 | static const struct riscv_pmu riscv_base_pmu = { | |
448 | .pmu = &min_pmu, | |
449 | .max_events = ARRAY_SIZE(riscv_hw_event_map), | |
450 | .map_hw_event = riscv_map_hw_event, | |
451 | .hw_events = riscv_hw_event_map, | |
452 | .map_cache_event = riscv_map_cache_event, | |
453 | .cache_events = &riscv_cache_event_map, | |
454 | .counter_width = 63, | |
455 | .num_counters = RISCV_BASE_COUNTERS + 0, | |
456 | .handle_irq = &riscv_base_pmu_handle_irq, | |
457 | ||
458 | /* This means this PMU has no IRQ. */ | |
459 | .irq = -1, | |
460 | }; | |
461 | ||
462 | static const struct of_device_id riscv_pmu_of_ids[] = { | |
463 | {.compatible = "riscv,base-pmu", .data = &riscv_base_pmu}, | |
464 | { /* sentinel value */ } | |
465 | }; | |
466 | ||
e7b146a8 | 467 | static int __init init_hw_perf_events(void) |
178e9fc4 AK |
468 | { |
469 | struct device_node *node = of_find_node_by_type(NULL, "pmu"); | |
470 | const struct of_device_id *of_id; | |
471 | ||
472 | riscv_pmu = &riscv_base_pmu; | |
473 | ||
474 | if (node) { | |
475 | of_id = of_match_node(riscv_pmu_of_ids, node); | |
476 | ||
477 | if (of_id) | |
478 | riscv_pmu = of_id->data; | |
94f9bf11 | 479 | of_node_put(node); |
178e9fc4 AK |
480 | } |
481 | ||
482 | perf_pmu_register(riscv_pmu->pmu, "cpu", PERF_TYPE_RAW); | |
483 | return 0; | |
484 | } | |
485 | arch_initcall(init_hw_perf_events); |