1 // SPDX-License-Identifier: GPL-2.0-only
3 * linux/kernel/profile.c
4 * Simple profiling. Manages a direct-mapped profile hit count buffer,
5 * with configurable resolution, support for restricting the cpus on
6 * which profiling is done, and switching between cpu time and
7 * schedule() calls via kernel command line parameters passed at boot.
9 * Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
11 * Consolidation of architecture support code for profiling,
12 * Nadia Yvette Chambers, Oracle, July 2004
13 * Amortized hit count accounting via per-cpu open-addressed hashtables
14 * to resolve timer interrupt livelocks, Nadia Yvette Chambers,
18 #include <linux/export.h>
19 #include <linux/profile.h>
20 #include <linux/memblock.h>
21 #include <linux/notifier.h>
23 #include <linux/cpumask.h>
24 #include <linux/cpu.h>
25 #include <linux/highmem.h>
26 #include <linux/mutex.h>
27 #include <linux/slab.h>
28 #include <linux/vmalloc.h>
29 #include <linux/sched/stat.h>
31 #include <asm/sections.h>
32 #include <asm/irq_regs.h>
33 #include <asm/ptrace.h>
38 #define PROFILE_GRPSHIFT 3
39 #define PROFILE_GRPSZ (1 << PROFILE_GRPSHIFT)
40 #define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
41 #define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
43 static atomic_t
*prof_buffer
;
44 static unsigned long prof_len
;
45 static unsigned short int prof_shift
;
47 int prof_on __read_mostly
;
48 EXPORT_SYMBOL_GPL(prof_on
);
50 static cpumask_var_t prof_cpu_mask
;
51 #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
52 static DEFINE_PER_CPU(struct profile_hit
*[2], cpu_profile_hits
);
53 static DEFINE_PER_CPU(int, cpu_profile_flip
);
54 static DEFINE_MUTEX(profile_flip_mutex
);
55 #endif /* CONFIG_SMP */
57 int profile_setup(char *str
)
59 static const char schedstr
[] = "schedule";
60 static const char sleepstr
[] = "sleep";
61 static const char kvmstr
[] = "kvm";
62 const char *select
= NULL
;
65 if (!strncmp(str
, sleepstr
, strlen(sleepstr
))) {
66 #ifdef CONFIG_SCHEDSTATS
67 force_schedstat_enabled();
68 prof_on
= SLEEP_PROFILING
;
71 pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
72 #endif /* CONFIG_SCHEDSTATS */
73 } else if (!strncmp(str
, schedstr
, strlen(schedstr
))) {
74 prof_on
= SCHED_PROFILING
;
76 } else if (!strncmp(str
, kvmstr
, strlen(kvmstr
))) {
77 prof_on
= KVM_PROFILING
;
79 } else if (get_option(&str
, &par
)) {
80 prof_shift
= clamp(par
, 0, BITS_PER_LONG
- 1);
81 prof_on
= CPU_PROFILING
;
82 pr_info("kernel profiling enabled (shift: %u)\n",
87 if (str
[strlen(select
)] == ',')
88 str
+= strlen(select
) + 1;
89 if (get_option(&str
, &par
))
90 prof_shift
= clamp(par
, 0, BITS_PER_LONG
- 1);
91 pr_info("kernel %s profiling enabled (shift: %u)\n",
97 __setup("profile=", profile_setup
);
100 int __ref
profile_init(void)
106 /* only text is profiled */
107 prof_len
= (_etext
- _stext
) >> prof_shift
;
110 pr_warn("profiling shift: %u too large\n", prof_shift
);
115 buffer_bytes
= prof_len
*sizeof(atomic_t
);
117 if (!alloc_cpumask_var(&prof_cpu_mask
, GFP_KERNEL
))
120 cpumask_copy(prof_cpu_mask
, cpu_possible_mask
);
122 prof_buffer
= kzalloc(buffer_bytes
, GFP_KERNEL
|__GFP_NOWARN
);
126 prof_buffer
= alloc_pages_exact(buffer_bytes
,
127 GFP_KERNEL
|__GFP_ZERO
|__GFP_NOWARN
);
131 prof_buffer
= vzalloc(buffer_bytes
);
135 free_cpumask_var(prof_cpu_mask
);
139 #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
141 * Each cpu has a pair of open-addressed hashtables for pending
142 * profile hits. read_profile() IPI's all cpus to request them
143 * to flip buffers and flushes their contents to prof_buffer itself.
144 * Flip requests are serialized by the profile_flip_mutex. The sole
145 * use of having a second hashtable is for avoiding cacheline
146 * contention that would otherwise happen during flushes of pending
147 * profile hits required for the accuracy of reported profile hits
148 * and so resurrect the interrupt livelock issue.
150 * The open-addressed hashtables are indexed by profile buffer slot
151 * and hold the number of pending hits to that profile buffer slot on
152 * a cpu in an entry. When the hashtable overflows, all pending hits
153 * are accounted to their corresponding profile buffer slots with
154 * atomic_add() and the hashtable emptied. As numerous pending hits
155 * may be accounted to a profile buffer slot in a hashtable entry,
156 * this amortizes a number of atomic profile buffer increments likely
157 * to be far larger than the number of entries in the hashtable,
158 * particularly given that the number of distinct profile buffer
159 * positions to which hits are accounted during short intervals (e.g.
160 * several seconds) is usually very small. Exclusion from buffer
161 * flipping is provided by interrupt disablement (note that for
162 * SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
164 * The hash function is meant to be lightweight as opposed to strong,
165 * and was vaguely inspired by ppc64 firmware-supported inverted
166 * pagetable hash functions, but uses a full hashtable full of finite
167 * collision chains, not just pairs of them.
171 static void __profile_flip_buffers(void *unused
)
173 int cpu
= smp_processor_id();
175 per_cpu(cpu_profile_flip
, cpu
) = !per_cpu(cpu_profile_flip
, cpu
);
178 static void profile_flip_buffers(void)
182 mutex_lock(&profile_flip_mutex
);
183 j
= per_cpu(cpu_profile_flip
, get_cpu());
185 on_each_cpu(__profile_flip_buffers
, NULL
, 1);
186 for_each_online_cpu(cpu
) {
187 struct profile_hit
*hits
= per_cpu(cpu_profile_hits
, cpu
)[j
];
188 for (i
= 0; i
< NR_PROFILE_HIT
; ++i
) {
194 atomic_add(hits
[i
].hits
, &prof_buffer
[hits
[i
].pc
]);
195 hits
[i
].hits
= hits
[i
].pc
= 0;
198 mutex_unlock(&profile_flip_mutex
);
201 static void profile_discard_flip_buffers(void)
205 mutex_lock(&profile_flip_mutex
);
206 i
= per_cpu(cpu_profile_flip
, get_cpu());
208 on_each_cpu(__profile_flip_buffers
, NULL
, 1);
209 for_each_online_cpu(cpu
) {
210 struct profile_hit
*hits
= per_cpu(cpu_profile_hits
, cpu
)[i
];
211 memset(hits
, 0, NR_PROFILE_HIT
*sizeof(struct profile_hit
));
213 mutex_unlock(&profile_flip_mutex
);
216 static void do_profile_hits(int type
, void *__pc
, unsigned int nr_hits
)
218 unsigned long primary
, secondary
, flags
, pc
= (unsigned long)__pc
;
220 struct profile_hit
*hits
;
222 pc
= min((pc
- (unsigned long)_stext
) >> prof_shift
, prof_len
- 1);
223 i
= primary
= (pc
& (NR_PROFILE_GRP
- 1)) << PROFILE_GRPSHIFT
;
224 secondary
= (~(pc
<< 1) & (NR_PROFILE_GRP
- 1)) << PROFILE_GRPSHIFT
;
226 hits
= per_cpu(cpu_profile_hits
, cpu
)[per_cpu(cpu_profile_flip
, cpu
)];
232 * We buffer the global profiler buffer into a per-CPU
233 * queue and thus reduce the number of global (and possibly
234 * NUMA-alien) accesses. The write-queue is self-coalescing:
236 local_irq_save(flags
);
238 for (j
= 0; j
< PROFILE_GRPSZ
; ++j
) {
239 if (hits
[i
+ j
].pc
== pc
) {
240 hits
[i
+ j
].hits
+= nr_hits
;
242 } else if (!hits
[i
+ j
].hits
) {
244 hits
[i
+ j
].hits
= nr_hits
;
248 i
= (i
+ secondary
) & (NR_PROFILE_HIT
- 1);
249 } while (i
!= primary
);
252 * Add the current hit(s) and flush the write-queue out
253 * to the global buffer:
255 atomic_add(nr_hits
, &prof_buffer
[pc
]);
256 for (i
= 0; i
< NR_PROFILE_HIT
; ++i
) {
257 atomic_add(hits
[i
].hits
, &prof_buffer
[hits
[i
].pc
]);
258 hits
[i
].pc
= hits
[i
].hits
= 0;
261 local_irq_restore(flags
);
265 static int profile_dead_cpu(unsigned int cpu
)
270 if (cpumask_available(prof_cpu_mask
))
271 cpumask_clear_cpu(cpu
, prof_cpu_mask
);
273 for (i
= 0; i
< 2; i
++) {
274 if (per_cpu(cpu_profile_hits
, cpu
)[i
]) {
275 page
= virt_to_page(per_cpu(cpu_profile_hits
, cpu
)[i
]);
276 per_cpu(cpu_profile_hits
, cpu
)[i
] = NULL
;
283 static int profile_prepare_cpu(unsigned int cpu
)
285 int i
, node
= cpu_to_mem(cpu
);
288 per_cpu(cpu_profile_flip
, cpu
) = 0;
290 for (i
= 0; i
< 2; i
++) {
291 if (per_cpu(cpu_profile_hits
, cpu
)[i
])
294 page
= __alloc_pages_node(node
, GFP_KERNEL
| __GFP_ZERO
, 0);
296 profile_dead_cpu(cpu
);
299 per_cpu(cpu_profile_hits
, cpu
)[i
] = page_address(page
);
305 static int profile_online_cpu(unsigned int cpu
)
307 if (cpumask_available(prof_cpu_mask
))
308 cpumask_set_cpu(cpu
, prof_cpu_mask
);
313 #else /* !CONFIG_SMP */
314 #define profile_flip_buffers() do { } while (0)
315 #define profile_discard_flip_buffers() do { } while (0)
317 static void do_profile_hits(int type
, void *__pc
, unsigned int nr_hits
)
320 pc
= ((unsigned long)__pc
- (unsigned long)_stext
) >> prof_shift
;
321 atomic_add(nr_hits
, &prof_buffer
[min(pc
, prof_len
- 1)]);
323 #endif /* !CONFIG_SMP */
325 void profile_hits(int type
, void *__pc
, unsigned int nr_hits
)
327 if (prof_on
!= type
|| !prof_buffer
)
329 do_profile_hits(type
, __pc
, nr_hits
);
331 EXPORT_SYMBOL_GPL(profile_hits
);
333 void profile_tick(int type
)
335 struct pt_regs
*regs
= get_irq_regs();
337 if (!user_mode(regs
) && cpumask_available(prof_cpu_mask
) &&
338 cpumask_test_cpu(smp_processor_id(), prof_cpu_mask
))
339 profile_hit(type
, (void *)profile_pc(regs
));
342 #ifdef CONFIG_PROC_FS
343 #include <linux/proc_fs.h>
344 #include <linux/seq_file.h>
345 #include <linux/uaccess.h>
347 static int prof_cpu_mask_proc_show(struct seq_file
*m
, void *v
)
349 seq_printf(m
, "%*pb\n", cpumask_pr_args(prof_cpu_mask
));
353 static int prof_cpu_mask_proc_open(struct inode
*inode
, struct file
*file
)
355 return single_open(file
, prof_cpu_mask_proc_show
, NULL
);
358 static ssize_t
prof_cpu_mask_proc_write(struct file
*file
,
359 const char __user
*buffer
, size_t count
, loff_t
*pos
)
361 cpumask_var_t new_value
;
364 if (!zalloc_cpumask_var(&new_value
, GFP_KERNEL
))
367 err
= cpumask_parse_user(buffer
, count
, new_value
);
369 cpumask_copy(prof_cpu_mask
, new_value
);
372 free_cpumask_var(new_value
);
376 static const struct proc_ops prof_cpu_mask_proc_ops
= {
377 .proc_open
= prof_cpu_mask_proc_open
,
378 .proc_read
= seq_read
,
379 .proc_lseek
= seq_lseek
,
380 .proc_release
= single_release
,
381 .proc_write
= prof_cpu_mask_proc_write
,
384 void create_prof_cpu_mask(void)
386 /* create /proc/irq/prof_cpu_mask */
387 proc_create("irq/prof_cpu_mask", 0600, NULL
, &prof_cpu_mask_proc_ops
);
391 * This function accesses profiling information. The returned data is
392 * binary: the sampling step and the actual contents of the profile
393 * buffer. Use of the program readprofile is recommended in order to
394 * get meaningful info out of these data.
397 read_profile(struct file
*file
, char __user
*buf
, size_t count
, loff_t
*ppos
)
399 unsigned long p
= *ppos
;
402 unsigned long sample_step
= 1UL << prof_shift
;
404 profile_flip_buffers();
405 if (p
>= (prof_len
+1)*sizeof(unsigned int))
407 if (count
> (prof_len
+1)*sizeof(unsigned int) - p
)
408 count
= (prof_len
+1)*sizeof(unsigned int) - p
;
411 while (p
< sizeof(unsigned int) && count
> 0) {
412 if (put_user(*((char *)(&sample_step
)+p
), buf
))
414 buf
++; p
++; count
--; read
++;
416 pnt
= (char *)prof_buffer
+ p
- sizeof(atomic_t
);
417 if (copy_to_user(buf
, (void *)pnt
, count
))
424 /* default is to not implement this call */
425 int __weak
setup_profiling_timer(unsigned mult
)
431 * Writing to /proc/profile resets the counters
433 * Writing a 'profiling multiplier' value into it also re-sets the profiling
434 * interrupt frequency, on architectures that support this.
436 static ssize_t
write_profile(struct file
*file
, const char __user
*buf
,
437 size_t count
, loff_t
*ppos
)
440 if (count
== sizeof(int)) {
441 unsigned int multiplier
;
443 if (copy_from_user(&multiplier
, buf
, sizeof(int)))
446 if (setup_profiling_timer(multiplier
))
450 profile_discard_flip_buffers();
451 memset(prof_buffer
, 0, prof_len
* sizeof(atomic_t
));
455 static const struct proc_ops profile_proc_ops
= {
456 .proc_read
= read_profile
,
457 .proc_write
= write_profile
,
458 .proc_lseek
= default_llseek
,
461 int __ref
create_proc_profile(void)
463 struct proc_dir_entry
*entry
;
465 enum cpuhp_state online_state
;
473 err
= cpuhp_setup_state(CPUHP_PROFILE_PREPARE
, "PROFILE_PREPARE",
474 profile_prepare_cpu
, profile_dead_cpu
);
478 err
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
, "AP_PROFILE_ONLINE",
479 profile_online_cpu
, NULL
);
485 entry
= proc_create("profile", S_IWUSR
| S_IRUGO
,
486 NULL
, &profile_proc_ops
);
489 proc_set_size(entry
, (1 + prof_len
) * sizeof(atomic_t
));
494 cpuhp_remove_state(online_state
);
496 cpuhp_remove_state(CPUHP_PROFILE_PREPARE
);
500 subsys_initcall(create_proc_profile
);
501 #endif /* CONFIG_PROC_FS */