1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
7 * - Added format output of fields of the trace point.
8 * This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
12 #define pr_fmt(fmt) fmt
14 #include <linux/workqueue.h>
15 #include <linux/security.h>
16 #include <linux/spinlock.h>
17 #include <linux/kthread.h>
18 #include <linux/tracefs.h>
19 #include <linux/uaccess.h>
20 #include <linux/module.h>
21 #include <linux/ctype.h>
22 #include <linux/sort.h>
23 #include <linux/slab.h>
24 #include <linux/delay.h>
26 #include <trace/events/sched.h>
27 #include <trace/syscall.h>
29 #include <asm/setup.h>
31 #include "trace_output.h"
34 #define TRACE_SYSTEM "TRACE_SYSTEM"
36 DEFINE_MUTEX(event_mutex
);
38 LIST_HEAD(ftrace_events
);
39 static LIST_HEAD(ftrace_generic_fields
);
40 static LIST_HEAD(ftrace_common_fields
);
41 static bool eventdir_initialized
;
43 static LIST_HEAD(module_strings
);
45 struct module_string
{
46 struct list_head next
;
47 struct module
*module
;
51 #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
53 static struct kmem_cache
*field_cachep
;
54 static struct kmem_cache
*file_cachep
;
56 static inline int system_refcount(struct event_subsystem
*system
)
58 return system
->ref_count
;
61 static int system_refcount_inc(struct event_subsystem
*system
)
63 return system
->ref_count
++;
66 static int system_refcount_dec(struct event_subsystem
*system
)
68 return --system
->ref_count
;
71 /* Double loops, do not use break, only goto's work */
72 #define do_for_each_event_file(tr, file) \
73 list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
74 list_for_each_entry(file, &tr->events, list)
76 #define do_for_each_event_file_safe(tr, file) \
77 list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
78 struct trace_event_file *___n; \
79 list_for_each_entry_safe(file, ___n, &tr->events, list)
81 #define while_for_each_event_file() \
84 static struct ftrace_event_field
*
85 __find_event_field(struct list_head
*head
, char *name
)
87 struct ftrace_event_field
*field
;
89 list_for_each_entry(field
, head
, link
) {
90 if (!strcmp(field
->name
, name
))
97 struct ftrace_event_field
*
98 trace_find_event_field(struct trace_event_call
*call
, char *name
)
100 struct ftrace_event_field
*field
;
101 struct list_head
*head
;
103 head
= trace_get_fields(call
);
104 field
= __find_event_field(head
, name
);
108 field
= __find_event_field(&ftrace_generic_fields
, name
);
112 return __find_event_field(&ftrace_common_fields
, name
);
115 static int __trace_define_field(struct list_head
*head
, const char *type
,
116 const char *name
, int offset
, int size
,
117 int is_signed
, int filter_type
)
119 struct ftrace_event_field
*field
;
121 field
= kmem_cache_alloc(field_cachep
, GFP_TRACE
);
128 if (filter_type
== FILTER_OTHER
)
129 field
->filter_type
= filter_assign_type(type
);
131 field
->filter_type
= filter_type
;
133 field
->offset
= offset
;
135 field
->is_signed
= is_signed
;
137 list_add(&field
->link
, head
);
142 int trace_define_field(struct trace_event_call
*call
, const char *type
,
143 const char *name
, int offset
, int size
, int is_signed
,
146 struct list_head
*head
;
148 if (WARN_ON(!call
->class))
151 head
= trace_get_fields(call
);
152 return __trace_define_field(head
, type
, name
, offset
, size
,
153 is_signed
, filter_type
);
155 EXPORT_SYMBOL_GPL(trace_define_field
);
157 #define __generic_field(type, item, filter_type) \
158 ret = __trace_define_field(&ftrace_generic_fields, #type, \
159 #item, 0, 0, is_signed_type(type), \
164 #define __common_field(type, item) \
165 ret = __trace_define_field(&ftrace_common_fields, #type, \
167 offsetof(typeof(ent), item), \
169 is_signed_type(type), FILTER_OTHER); \
173 static int trace_define_generic_fields(void)
177 __generic_field(int, CPU
, FILTER_CPU
);
178 __generic_field(int, cpu
, FILTER_CPU
);
179 __generic_field(char *, COMM
, FILTER_COMM
);
180 __generic_field(char *, comm
, FILTER_COMM
);
185 static int trace_define_common_fields(void)
188 struct trace_entry ent
;
190 __common_field(unsigned short, type
);
191 __common_field(unsigned char, flags
);
192 /* Holds both preempt_count and migrate_disable */
193 __common_field(unsigned char, preempt_count
);
194 __common_field(int, pid
);
199 static void trace_destroy_fields(struct trace_event_call
*call
)
201 struct ftrace_event_field
*field
, *next
;
202 struct list_head
*head
;
204 head
= trace_get_fields(call
);
205 list_for_each_entry_safe(field
, next
, head
, link
) {
206 list_del(&field
->link
);
207 kmem_cache_free(field_cachep
, field
);
212 * run-time version of trace_event_get_offsets_<call>() that returns the last
213 * accessible offset of trace fields excluding __dynamic_array bytes
215 int trace_event_get_offsets(struct trace_event_call
*call
)
217 struct ftrace_event_field
*tail
;
218 struct list_head
*head
;
220 head
= trace_get_fields(call
);
222 * head->next points to the last field with the largest offset,
223 * since it was added last by trace_define_field()
225 tail
= list_first_entry(head
, struct ftrace_event_field
, link
);
226 return tail
->offset
+ tail
->size
;
230 * Check if the referenced field is an array and return true,
231 * as arrays are OK to dereference.
233 static bool test_field(const char *fmt
, struct trace_event_call
*call
)
235 struct trace_event_fields
*field
= call
->class->fields_array
;
236 const char *array_descriptor
;
240 if (!(len
= str_has_prefix(fmt
, "REC->")))
243 for (p
= fmt
; *p
; p
++) {
244 if (!isalnum(*p
) && *p
!= '_')
249 for (; field
->type
; field
++) {
250 if (strncmp(field
->name
, fmt
, len
) ||
253 array_descriptor
= strchr(field
->type
, '[');
254 /* This is an array and is OK to dereference. */
255 return array_descriptor
!= NULL
;
261 * Examine the print fmt of the event looking for unsafe dereference
262 * pointers using %p* that could be recorded in the trace event and
263 * much later referenced after the pointer was freed. Dereferencing
264 * pointers are OK, if it is dereferenced into the event itself.
266 static void test_event_printk(struct trace_event_call
*call
)
268 u64 dereference_flags
= 0;
270 const char *fmt
, *c
, *r
, *a
;
277 fmt
= call
->print_fmt
;
282 for (i
= 0; fmt
[i
]; i
++) {
292 * The print fmt starts with a string that
293 * is processed first to find %p* usage,
294 * then after the first string, the print fmt
295 * contains arguments that are used to check
296 * if the dereferenced %p* usage is safe.
305 * If there was no %p* uses
308 if (!dereference_flags
)
313 if (in_quote
== fmt
[i
])
320 if (!first
|| !in_quote
)
329 /* Find dereferencing fields */
330 switch (fmt
[i
+ 1]) {
331 case 'B': case 'R': case 'r':
332 case 'b': case 'M': case 'm':
333 case 'I': case 'i': case 'E':
334 case 'U': case 'V': case 'N':
335 case 'a': case 'd': case 'D':
336 case 'g': case 't': case 'C':
338 if (WARN_ONCE(arg
== 63,
339 "Too many args for event: %s",
340 trace_event_name(call
)))
342 dereference_flags
|= 1ULL << arg
;
350 /* Increment arg if %*s exists. */
351 for (j
= 0; fmt
[i
+ j
]; j
++) {
352 if (isdigit(fmt
[i
+ j
]) ||
355 if (fmt
[i
+ j
] == '*') {
359 if ((fmt
[i
+ j
] == 's') && star
)
378 if (WARN_ONCE(parens
< 0,
379 "Paren mismatch for event: %s\narg='%s'\n%*s",
380 trace_event_name(call
),
382 (i
- start_arg
) + 5, "^"))
386 if (in_quote
|| parens
)
389 while (isspace(fmt
[i
]))
392 if (!(dereference_flags
& (1ULL << arg
)))
395 /* Find the REC-> in the argument */
396 c
= strchr(fmt
+ i
, ',');
397 r
= strstr(fmt
+ i
, "REC->");
398 if (r
&& (!c
|| r
< c
)) {
400 * Addresses of events on the buffer,
401 * or an array on the buffer is
403 * There's ways to fool this, but
404 * this is to catch common mistakes,
405 * not malicious code.
407 a
= strchr(fmt
+ i
, '&');
408 if ((a
&& (a
< r
)) || test_field(r
, call
))
409 dereference_flags
&= ~(1ULL << arg
);
410 } else if ((r
= strstr(fmt
+ i
, "__get_dynamic_array(")) &&
412 dereference_flags
&= ~(1ULL << arg
);
413 } else if ((r
= strstr(fmt
+ i
, "__get_sockaddr(")) &&
415 dereference_flags
&= ~(1ULL << arg
);
425 * If you triggered the below warning, the trace event reported
426 * uses an unsafe dereference pointer %p*. As the data stored
427 * at the trace event time may no longer exist when the trace
428 * event is printed, dereferencing to the original source is
429 * unsafe. The source of the dereference must be copied into the
430 * event itself, and the dereference must access the copy instead.
432 if (WARN_ON_ONCE(dereference_flags
)) {
434 while (!(dereference_flags
& 1)) {
435 dereference_flags
>>= 1;
438 pr_warn("event %s has unsafe dereference of argument %d\n",
439 trace_event_name(call
), arg
);
440 pr_warn("print_fmt: %s\n", fmt
);
444 int trace_event_raw_init(struct trace_event_call
*call
)
448 id
= register_trace_event(&call
->event
);
452 test_event_printk(call
);
456 EXPORT_SYMBOL_GPL(trace_event_raw_init
);
458 bool trace_event_ignore_this_pid(struct trace_event_file
*trace_file
)
460 struct trace_array
*tr
= trace_file
->tr
;
461 struct trace_array_cpu
*data
;
462 struct trace_pid_list
*no_pid_list
;
463 struct trace_pid_list
*pid_list
;
465 pid_list
= rcu_dereference_raw(tr
->filtered_pids
);
466 no_pid_list
= rcu_dereference_raw(tr
->filtered_no_pids
);
468 if (!pid_list
&& !no_pid_list
)
471 data
= this_cpu_ptr(tr
->array_buffer
.data
);
473 return data
->ignore_pid
;
475 EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid
);
477 void *trace_event_buffer_reserve(struct trace_event_buffer
*fbuffer
,
478 struct trace_event_file
*trace_file
,
481 struct trace_event_call
*event_call
= trace_file
->event_call
;
483 if ((trace_file
->flags
& EVENT_FILE_FL_PID_FILTER
) &&
484 trace_event_ignore_this_pid(trace_file
))
488 * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
489 * preemption (adding one to the preempt_count). Since we are
490 * interested in the preempt_count at the time the tracepoint was
491 * hit, we need to subtract one to offset the increment.
493 fbuffer
->trace_ctx
= tracing_gen_ctx_dec();
494 fbuffer
->trace_file
= trace_file
;
497 trace_event_buffer_lock_reserve(&fbuffer
->buffer
, trace_file
,
498 event_call
->event
.type
, len
,
503 fbuffer
->regs
= NULL
;
504 fbuffer
->entry
= ring_buffer_event_data(fbuffer
->event
);
505 return fbuffer
->entry
;
507 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve
);
509 int trace_event_reg(struct trace_event_call
*call
,
510 enum trace_reg type
, void *data
)
512 struct trace_event_file
*file
= data
;
514 WARN_ON(!(call
->flags
& TRACE_EVENT_FL_TRACEPOINT
));
516 case TRACE_REG_REGISTER
:
517 return tracepoint_probe_register(call
->tp
,
520 case TRACE_REG_UNREGISTER
:
521 tracepoint_probe_unregister(call
->tp
,
526 #ifdef CONFIG_PERF_EVENTS
527 case TRACE_REG_PERF_REGISTER
:
528 return tracepoint_probe_register(call
->tp
,
529 call
->class->perf_probe
,
531 case TRACE_REG_PERF_UNREGISTER
:
532 tracepoint_probe_unregister(call
->tp
,
533 call
->class->perf_probe
,
536 case TRACE_REG_PERF_OPEN
:
537 case TRACE_REG_PERF_CLOSE
:
538 case TRACE_REG_PERF_ADD
:
539 case TRACE_REG_PERF_DEL
:
545 EXPORT_SYMBOL_GPL(trace_event_reg
);
547 void trace_event_enable_cmd_record(bool enable
)
549 struct trace_event_file
*file
;
550 struct trace_array
*tr
;
552 lockdep_assert_held(&event_mutex
);
554 do_for_each_event_file(tr
, file
) {
556 if (!(file
->flags
& EVENT_FILE_FL_ENABLED
))
560 tracing_start_cmdline_record();
561 set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT
, &file
->flags
);
563 tracing_stop_cmdline_record();
564 clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT
, &file
->flags
);
566 } while_for_each_event_file();
569 void trace_event_enable_tgid_record(bool enable
)
571 struct trace_event_file
*file
;
572 struct trace_array
*tr
;
574 lockdep_assert_held(&event_mutex
);
576 do_for_each_event_file(tr
, file
) {
577 if (!(file
->flags
& EVENT_FILE_FL_ENABLED
))
581 tracing_start_tgid_record();
582 set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT
, &file
->flags
);
584 tracing_stop_tgid_record();
585 clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT
,
588 } while_for_each_event_file();
591 static int __ftrace_event_enable_disable(struct trace_event_file
*file
,
592 int enable
, int soft_disable
)
594 struct trace_event_call
*call
= file
->event_call
;
595 struct trace_array
*tr
= file
->tr
;
596 unsigned long file_flags
= file
->flags
;
603 * When soft_disable is set and enable is cleared, the sm_ref
604 * reference counter is decremented. If it reaches 0, we want
605 * to clear the SOFT_DISABLED flag but leave the event in the
606 * state that it was. That is, if the event was enabled and
607 * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
608 * is set we do not want the event to be enabled before we
611 * When soft_disable is not set but the SOFT_MODE flag is,
612 * we do nothing. Do not disable the tracepoint, otherwise
613 * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
616 if (atomic_dec_return(&file
->sm_ref
) > 0)
618 disable
= file
->flags
& EVENT_FILE_FL_SOFT_DISABLED
;
619 clear_bit(EVENT_FILE_FL_SOFT_MODE_BIT
, &file
->flags
);
621 disable
= !(file
->flags
& EVENT_FILE_FL_SOFT_MODE
);
623 if (disable
&& (file
->flags
& EVENT_FILE_FL_ENABLED
)) {
624 clear_bit(EVENT_FILE_FL_ENABLED_BIT
, &file
->flags
);
625 if (file
->flags
& EVENT_FILE_FL_RECORDED_CMD
) {
626 tracing_stop_cmdline_record();
627 clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT
, &file
->flags
);
630 if (file
->flags
& EVENT_FILE_FL_RECORDED_TGID
) {
631 tracing_stop_tgid_record();
632 clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT
, &file
->flags
);
635 call
->class->reg(call
, TRACE_REG_UNREGISTER
, file
);
637 /* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
638 if (file
->flags
& EVENT_FILE_FL_SOFT_MODE
)
639 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT
, &file
->flags
);
641 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT
, &file
->flags
);
645 * When soft_disable is set and enable is set, we want to
646 * register the tracepoint for the event, but leave the event
647 * as is. That means, if the event was already enabled, we do
648 * nothing (but set SOFT_MODE). If the event is disabled, we
649 * set SOFT_DISABLED before enabling the event tracepoint, so
650 * it still seems to be disabled.
653 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT
, &file
->flags
);
655 if (atomic_inc_return(&file
->sm_ref
) > 1)
657 set_bit(EVENT_FILE_FL_SOFT_MODE_BIT
, &file
->flags
);
660 if (!(file
->flags
& EVENT_FILE_FL_ENABLED
)) {
661 bool cmd
= false, tgid
= false;
663 /* Keep the event disabled, when going to SOFT_MODE. */
665 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT
, &file
->flags
);
667 if (tr
->trace_flags
& TRACE_ITER_RECORD_CMD
) {
669 tracing_start_cmdline_record();
670 set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT
, &file
->flags
);
673 if (tr
->trace_flags
& TRACE_ITER_RECORD_TGID
) {
675 tracing_start_tgid_record();
676 set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT
, &file
->flags
);
679 ret
= call
->class->reg(call
, TRACE_REG_REGISTER
, file
);
682 tracing_stop_cmdline_record();
684 tracing_stop_tgid_record();
685 pr_info("event trace: Could not enable event "
686 "%s\n", trace_event_name(call
));
689 set_bit(EVENT_FILE_FL_ENABLED_BIT
, &file
->flags
);
691 /* WAS_ENABLED gets set but never cleared. */
692 set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT
, &file
->flags
);
697 /* Enable or disable use of trace_buffered_event */
698 if ((file_flags
& EVENT_FILE_FL_SOFT_DISABLED
) !=
699 (file
->flags
& EVENT_FILE_FL_SOFT_DISABLED
)) {
700 if (file
->flags
& EVENT_FILE_FL_SOFT_DISABLED
)
701 trace_buffered_event_enable();
703 trace_buffered_event_disable();
709 int trace_event_enable_disable(struct trace_event_file
*file
,
710 int enable
, int soft_disable
)
712 return __ftrace_event_enable_disable(file
, enable
, soft_disable
);
715 static int ftrace_event_enable_disable(struct trace_event_file
*file
,
718 return __ftrace_event_enable_disable(file
, enable
, 0);
721 static void ftrace_clear_events(struct trace_array
*tr
)
723 struct trace_event_file
*file
;
725 mutex_lock(&event_mutex
);
726 list_for_each_entry(file
, &tr
->events
, list
) {
727 ftrace_event_enable_disable(file
, 0);
729 mutex_unlock(&event_mutex
);
733 event_filter_pid_sched_process_exit(void *data
, struct task_struct
*task
)
735 struct trace_pid_list
*pid_list
;
736 struct trace_array
*tr
= data
;
738 pid_list
= rcu_dereference_raw(tr
->filtered_pids
);
739 trace_filter_add_remove_task(pid_list
, NULL
, task
);
741 pid_list
= rcu_dereference_raw(tr
->filtered_no_pids
);
742 trace_filter_add_remove_task(pid_list
, NULL
, task
);
746 event_filter_pid_sched_process_fork(void *data
,
747 struct task_struct
*self
,
748 struct task_struct
*task
)
750 struct trace_pid_list
*pid_list
;
751 struct trace_array
*tr
= data
;
753 pid_list
= rcu_dereference_sched(tr
->filtered_pids
);
754 trace_filter_add_remove_task(pid_list
, self
, task
);
756 pid_list
= rcu_dereference_sched(tr
->filtered_no_pids
);
757 trace_filter_add_remove_task(pid_list
, self
, task
);
760 void trace_event_follow_fork(struct trace_array
*tr
, bool enable
)
763 register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork
,
765 register_trace_prio_sched_process_free(event_filter_pid_sched_process_exit
,
768 unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork
,
770 unregister_trace_sched_process_free(event_filter_pid_sched_process_exit
,
776 event_filter_pid_sched_switch_probe_pre(void *data
, bool preempt
,
777 struct task_struct
*prev
,
778 struct task_struct
*next
,
779 unsigned int prev_state
)
781 struct trace_array
*tr
= data
;
782 struct trace_pid_list
*no_pid_list
;
783 struct trace_pid_list
*pid_list
;
786 pid_list
= rcu_dereference_sched(tr
->filtered_pids
);
787 no_pid_list
= rcu_dereference_sched(tr
->filtered_no_pids
);
790 * Sched switch is funny, as we only want to ignore it
791 * in the notrace case if both prev and next should be ignored.
793 ret
= trace_ignore_this_task(NULL
, no_pid_list
, prev
) &&
794 trace_ignore_this_task(NULL
, no_pid_list
, next
);
796 this_cpu_write(tr
->array_buffer
.data
->ignore_pid
, ret
||
797 (trace_ignore_this_task(pid_list
, NULL
, prev
) &&
798 trace_ignore_this_task(pid_list
, NULL
, next
)));
802 event_filter_pid_sched_switch_probe_post(void *data
, bool preempt
,
803 struct task_struct
*prev
,
804 struct task_struct
*next
,
805 unsigned int prev_state
)
807 struct trace_array
*tr
= data
;
808 struct trace_pid_list
*no_pid_list
;
809 struct trace_pid_list
*pid_list
;
811 pid_list
= rcu_dereference_sched(tr
->filtered_pids
);
812 no_pid_list
= rcu_dereference_sched(tr
->filtered_no_pids
);
814 this_cpu_write(tr
->array_buffer
.data
->ignore_pid
,
815 trace_ignore_this_task(pid_list
, no_pid_list
, next
));
819 event_filter_pid_sched_wakeup_probe_pre(void *data
, struct task_struct
*task
)
821 struct trace_array
*tr
= data
;
822 struct trace_pid_list
*no_pid_list
;
823 struct trace_pid_list
*pid_list
;
825 /* Nothing to do if we are already tracing */
826 if (!this_cpu_read(tr
->array_buffer
.data
->ignore_pid
))
829 pid_list
= rcu_dereference_sched(tr
->filtered_pids
);
830 no_pid_list
= rcu_dereference_sched(tr
->filtered_no_pids
);
832 this_cpu_write(tr
->array_buffer
.data
->ignore_pid
,
833 trace_ignore_this_task(pid_list
, no_pid_list
, task
));
837 event_filter_pid_sched_wakeup_probe_post(void *data
, struct task_struct
*task
)
839 struct trace_array
*tr
= data
;
840 struct trace_pid_list
*no_pid_list
;
841 struct trace_pid_list
*pid_list
;
843 /* Nothing to do if we are not tracing */
844 if (this_cpu_read(tr
->array_buffer
.data
->ignore_pid
))
847 pid_list
= rcu_dereference_sched(tr
->filtered_pids
);
848 no_pid_list
= rcu_dereference_sched(tr
->filtered_no_pids
);
850 /* Set tracing if current is enabled */
851 this_cpu_write(tr
->array_buffer
.data
->ignore_pid
,
852 trace_ignore_this_task(pid_list
, no_pid_list
, current
));
855 static void unregister_pid_events(struct trace_array
*tr
)
857 unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre
, tr
);
858 unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post
, tr
);
860 unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre
, tr
);
861 unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post
, tr
);
863 unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre
, tr
);
864 unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post
, tr
);
866 unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre
, tr
);
867 unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post
, tr
);
870 static void __ftrace_clear_event_pids(struct trace_array
*tr
, int type
)
872 struct trace_pid_list
*pid_list
;
873 struct trace_pid_list
*no_pid_list
;
874 struct trace_event_file
*file
;
877 pid_list
= rcu_dereference_protected(tr
->filtered_pids
,
878 lockdep_is_held(&event_mutex
));
879 no_pid_list
= rcu_dereference_protected(tr
->filtered_no_pids
,
880 lockdep_is_held(&event_mutex
));
882 /* Make sure there's something to do */
883 if (!pid_type_enabled(type
, pid_list
, no_pid_list
))
886 if (!still_need_pid_events(type
, pid_list
, no_pid_list
)) {
887 unregister_pid_events(tr
);
889 list_for_each_entry(file
, &tr
->events
, list
) {
890 clear_bit(EVENT_FILE_FL_PID_FILTER_BIT
, &file
->flags
);
893 for_each_possible_cpu(cpu
)
894 per_cpu_ptr(tr
->array_buffer
.data
, cpu
)->ignore_pid
= false;
897 if (type
& TRACE_PIDS
)
898 rcu_assign_pointer(tr
->filtered_pids
, NULL
);
900 if (type
& TRACE_NO_PIDS
)
901 rcu_assign_pointer(tr
->filtered_no_pids
, NULL
);
903 /* Wait till all users are no longer using pid filtering */
904 tracepoint_synchronize_unregister();
906 if ((type
& TRACE_PIDS
) && pid_list
)
907 trace_pid_list_free(pid_list
);
909 if ((type
& TRACE_NO_PIDS
) && no_pid_list
)
910 trace_pid_list_free(no_pid_list
);
913 static void ftrace_clear_event_pids(struct trace_array
*tr
, int type
)
915 mutex_lock(&event_mutex
);
916 __ftrace_clear_event_pids(tr
, type
);
917 mutex_unlock(&event_mutex
);
920 static void __put_system(struct event_subsystem
*system
)
922 struct event_filter
*filter
= system
->filter
;
924 WARN_ON_ONCE(system_refcount(system
) == 0);
925 if (system_refcount_dec(system
))
928 list_del(&system
->list
);
931 kfree(filter
->filter_string
);
934 kfree_const(system
->name
);
938 static void __get_system(struct event_subsystem
*system
)
940 WARN_ON_ONCE(system_refcount(system
) == 0);
941 system_refcount_inc(system
);
944 static void __get_system_dir(struct trace_subsystem_dir
*dir
)
946 WARN_ON_ONCE(dir
->ref_count
== 0);
948 __get_system(dir
->subsystem
);
951 static void __put_system_dir(struct trace_subsystem_dir
*dir
)
953 WARN_ON_ONCE(dir
->ref_count
== 0);
954 /* If the subsystem is about to be freed, the dir must be too */
955 WARN_ON_ONCE(system_refcount(dir
->subsystem
) == 1 && dir
->ref_count
!= 1);
957 __put_system(dir
->subsystem
);
958 if (!--dir
->ref_count
)
962 static void put_system(struct trace_subsystem_dir
*dir
)
964 mutex_lock(&event_mutex
);
965 __put_system_dir(dir
);
966 mutex_unlock(&event_mutex
);
969 static void remove_subsystem(struct trace_subsystem_dir
*dir
)
974 if (!--dir
->nr_events
) {
975 tracefs_remove(dir
->entry
);
976 list_del(&dir
->list
);
977 __put_system_dir(dir
);
981 static void remove_event_file_dir(struct trace_event_file
*file
)
983 struct dentry
*dir
= file
->dir
;
984 struct dentry
*child
;
987 spin_lock(&dir
->d_lock
); /* probably unneeded */
988 list_for_each_entry(child
, &dir
->d_subdirs
, d_child
) {
989 if (d_really_is_positive(child
)) /* probably unneeded */
990 d_inode(child
)->i_private
= NULL
;
992 spin_unlock(&dir
->d_lock
);
997 list_del(&file
->list
);
998 remove_subsystem(file
->system
);
999 free_event_filter(file
->filter
);
1000 kmem_cache_free(file_cachep
, file
);
1004 * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
1007 __ftrace_set_clr_event_nolock(struct trace_array
*tr
, const char *match
,
1008 const char *sub
, const char *event
, int set
)
1010 struct trace_event_file
*file
;
1011 struct trace_event_call
*call
;
1016 list_for_each_entry(file
, &tr
->events
, list
) {
1018 call
= file
->event_call
;
1019 name
= trace_event_name(call
);
1021 if (!name
|| !call
->class || !call
->class->reg
)
1024 if (call
->flags
& TRACE_EVENT_FL_IGNORE_ENABLE
)
1028 strcmp(match
, name
) != 0 &&
1029 strcmp(match
, call
->class->system
) != 0)
1032 if (sub
&& strcmp(sub
, call
->class->system
) != 0)
1035 if (event
&& strcmp(event
, name
) != 0)
1038 ret
= ftrace_event_enable_disable(file
, set
);
1041 * Save the first error and return that. Some events
1042 * may still have been enabled, but let the user
1043 * know that something went wrong.
1054 static int __ftrace_set_clr_event(struct trace_array
*tr
, const char *match
,
1055 const char *sub
, const char *event
, int set
)
1059 mutex_lock(&event_mutex
);
1060 ret
= __ftrace_set_clr_event_nolock(tr
, match
, sub
, event
, set
);
1061 mutex_unlock(&event_mutex
);
1066 int ftrace_set_clr_event(struct trace_array
*tr
, char *buf
, int set
)
1068 char *event
= NULL
, *sub
= NULL
, *match
;
1074 * The buf format can be <subsystem>:<event-name>
1075 * *:<event-name> means any event by that name.
1076 * :<event-name> is the same.
1078 * <subsystem>:* means all events in that subsystem
1079 * <subsystem>: means the same.
1081 * <name> (no ':') means all events in a subsystem with
1082 * the name <name> or any event that matches <name>
1085 match
= strsep(&buf
, ":");
1091 if (!strlen(sub
) || strcmp(sub
, "*") == 0)
1093 if (!strlen(event
) || strcmp(event
, "*") == 0)
1097 ret
= __ftrace_set_clr_event(tr
, match
, sub
, event
, set
);
1099 /* Put back the colon to allow this to be called again */
1107 * trace_set_clr_event - enable or disable an event
1108 * @system: system name to match (NULL for any system)
1109 * @event: event name to match (NULL for all events, within system)
1110 * @set: 1 to enable, 0 to disable
1112 * This is a way for other parts of the kernel to enable or disable
1115 * Returns 0 on success, -EINVAL if the parameters do not match any
1116 * registered events.
1118 int trace_set_clr_event(const char *system
, const char *event
, int set
)
1120 struct trace_array
*tr
= top_trace_array();
1125 return __ftrace_set_clr_event(tr
, NULL
, system
, event
, set
);
1127 EXPORT_SYMBOL_GPL(trace_set_clr_event
);
1130 * trace_array_set_clr_event - enable or disable an event for a trace array.
1131 * @tr: concerned trace array.
1132 * @system: system name to match (NULL for any system)
1133 * @event: event name to match (NULL for all events, within system)
1134 * @enable: true to enable, false to disable
1136 * This is a way for other parts of the kernel to enable or disable
1139 * Returns 0 on success, -EINVAL if the parameters do not match any
1140 * registered events.
1142 int trace_array_set_clr_event(struct trace_array
*tr
, const char *system
,
1143 const char *event
, bool enable
)
1150 set
= (enable
== true) ? 1 : 0;
1151 return __ftrace_set_clr_event(tr
, NULL
, system
, event
, set
);
1153 EXPORT_SYMBOL_GPL(trace_array_set_clr_event
);
1155 /* 128 should be much more than enough */
1156 #define EVENT_BUF_SIZE 127
1159 ftrace_event_write(struct file
*file
, const char __user
*ubuf
,
1160 size_t cnt
, loff_t
*ppos
)
1162 struct trace_parser parser
;
1163 struct seq_file
*m
= file
->private_data
;
1164 struct trace_array
*tr
= m
->private;
1170 ret
= tracing_update_buffers();
1174 if (trace_parser_get_init(&parser
, EVENT_BUF_SIZE
+ 1))
1177 read
= trace_get_user(&parser
, ubuf
, cnt
, ppos
);
1179 if (read
>= 0 && trace_parser_loaded((&parser
))) {
1182 if (*parser
.buffer
== '!')
1185 ret
= ftrace_set_clr_event(tr
, parser
.buffer
+ !set
, set
);
1193 trace_parser_put(&parser
);
1199 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1201 struct trace_event_file
*file
= v
;
1202 struct trace_event_call
*call
;
1203 struct trace_array
*tr
= m
->private;
1207 list_for_each_entry_continue(file
, &tr
->events
, list
) {
1208 call
= file
->event_call
;
1210 * The ftrace subsystem is for showing formats only.
1211 * They can not be enabled or disabled via the event files.
1213 if (call
->class && call
->class->reg
&&
1214 !(call
->flags
& TRACE_EVENT_FL_IGNORE_ENABLE
))
1221 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
1223 struct trace_event_file
*file
;
1224 struct trace_array
*tr
= m
->private;
1227 mutex_lock(&event_mutex
);
1229 file
= list_entry(&tr
->events
, struct trace_event_file
, list
);
1230 for (l
= 0; l
<= *pos
; ) {
1231 file
= t_next(m
, file
, &l
);
1239 s_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1241 struct trace_event_file
*file
= v
;
1242 struct trace_array
*tr
= m
->private;
1246 list_for_each_entry_continue(file
, &tr
->events
, list
) {
1247 if (file
->flags
& EVENT_FILE_FL_ENABLED
)
1254 static void *s_start(struct seq_file
*m
, loff_t
*pos
)
1256 struct trace_event_file
*file
;
1257 struct trace_array
*tr
= m
->private;
1260 mutex_lock(&event_mutex
);
1262 file
= list_entry(&tr
->events
, struct trace_event_file
, list
);
1263 for (l
= 0; l
<= *pos
; ) {
1264 file
= s_next(m
, file
, &l
);
1271 static int t_show(struct seq_file
*m
, void *v
)
1273 struct trace_event_file
*file
= v
;
1274 struct trace_event_call
*call
= file
->event_call
;
1276 if (strcmp(call
->class->system
, TRACE_SYSTEM
) != 0)
1277 seq_printf(m
, "%s:", call
->class->system
);
1278 seq_printf(m
, "%s\n", trace_event_name(call
));
1283 static void t_stop(struct seq_file
*m
, void *p
)
1285 mutex_unlock(&event_mutex
);
1289 __next(struct seq_file
*m
, void *v
, loff_t
*pos
, int type
)
1291 struct trace_array
*tr
= m
->private;
1292 struct trace_pid_list
*pid_list
;
1294 if (type
== TRACE_PIDS
)
1295 pid_list
= rcu_dereference_sched(tr
->filtered_pids
);
1297 pid_list
= rcu_dereference_sched(tr
->filtered_no_pids
);
1299 return trace_pid_next(pid_list
, v
, pos
);
1303 p_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1305 return __next(m
, v
, pos
, TRACE_PIDS
);
1309 np_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1311 return __next(m
, v
, pos
, TRACE_NO_PIDS
);
1314 static void *__start(struct seq_file
*m
, loff_t
*pos
, int type
)
1317 struct trace_pid_list
*pid_list
;
1318 struct trace_array
*tr
= m
->private;
1321 * Grab the mutex, to keep calls to p_next() having the same
1322 * tr->filtered_pids as p_start() has.
1323 * If we just passed the tr->filtered_pids around, then RCU would
1324 * have been enough, but doing that makes things more complex.
1326 mutex_lock(&event_mutex
);
1327 rcu_read_lock_sched();
1329 if (type
== TRACE_PIDS
)
1330 pid_list
= rcu_dereference_sched(tr
->filtered_pids
);
1332 pid_list
= rcu_dereference_sched(tr
->filtered_no_pids
);
1337 return trace_pid_start(pid_list
, pos
);
1340 static void *p_start(struct seq_file
*m
, loff_t
*pos
)
1343 return __start(m
, pos
, TRACE_PIDS
);
1346 static void *np_start(struct seq_file
*m
, loff_t
*pos
)
1349 return __start(m
, pos
, TRACE_NO_PIDS
);
1352 static void p_stop(struct seq_file
*m
, void *p
)
1355 rcu_read_unlock_sched();
1356 mutex_unlock(&event_mutex
);
1360 event_enable_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
1363 struct trace_event_file
*file
;
1364 unsigned long flags
;
1367 mutex_lock(&event_mutex
);
1368 file
= event_file_data(filp
);
1370 flags
= file
->flags
;
1371 mutex_unlock(&event_mutex
);
1376 if (flags
& EVENT_FILE_FL_ENABLED
&&
1377 !(flags
& EVENT_FILE_FL_SOFT_DISABLED
))
1380 if (flags
& EVENT_FILE_FL_SOFT_DISABLED
||
1381 flags
& EVENT_FILE_FL_SOFT_MODE
)
1386 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, strlen(buf
));
1390 event_enable_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
1393 struct trace_event_file
*file
;
1397 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
1401 ret
= tracing_update_buffers();
1409 mutex_lock(&event_mutex
);
1410 file
= event_file_data(filp
);
1412 ret
= ftrace_event_enable_disable(file
, val
);
1413 mutex_unlock(&event_mutex
);
1422 return ret
? ret
: cnt
;
1426 system_enable_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
1429 const char set_to_char
[4] = { '?', '0', '1', 'X' };
1430 struct trace_subsystem_dir
*dir
= filp
->private_data
;
1431 struct event_subsystem
*system
= dir
->subsystem
;
1432 struct trace_event_call
*call
;
1433 struct trace_event_file
*file
;
1434 struct trace_array
*tr
= dir
->tr
;
1439 mutex_lock(&event_mutex
);
1440 list_for_each_entry(file
, &tr
->events
, list
) {
1441 call
= file
->event_call
;
1442 if ((call
->flags
& TRACE_EVENT_FL_IGNORE_ENABLE
) ||
1443 !trace_event_name(call
) || !call
->class || !call
->class->reg
)
1446 if (system
&& strcmp(call
->class->system
, system
->name
) != 0)
1450 * We need to find out if all the events are set
1451 * or if all events or cleared, or if we have
1454 set
|= (1 << !!(file
->flags
& EVENT_FILE_FL_ENABLED
));
1457 * If we have a mixture, no need to look further.
1462 mutex_unlock(&event_mutex
);
1464 buf
[0] = set_to_char
[set
];
1467 ret
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
1473 system_enable_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
1476 struct trace_subsystem_dir
*dir
= filp
->private_data
;
1477 struct event_subsystem
*system
= dir
->subsystem
;
1478 const char *name
= NULL
;
1482 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
1486 ret
= tracing_update_buffers();
1490 if (val
!= 0 && val
!= 1)
1494 * Opening of "enable" adds a ref count to system,
1495 * so the name is safe to use.
1498 name
= system
->name
;
1500 ret
= __ftrace_set_clr_event(dir
->tr
, NULL
, name
, NULL
, val
);
1514 FORMAT_FIELD_SEPERATOR
= 2,
1515 FORMAT_PRINTFMT
= 3,
1518 static void *f_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1520 struct trace_event_call
*call
= event_file_data(m
->private);
1521 struct list_head
*common_head
= &ftrace_common_fields
;
1522 struct list_head
*head
= trace_get_fields(call
);
1523 struct list_head
*node
= v
;
1527 switch ((unsigned long)v
) {
1532 case FORMAT_FIELD_SEPERATOR
:
1536 case FORMAT_PRINTFMT
:
1542 if (node
== common_head
)
1543 return (void *)FORMAT_FIELD_SEPERATOR
;
1544 else if (node
== head
)
1545 return (void *)FORMAT_PRINTFMT
;
1550 static int f_show(struct seq_file
*m
, void *v
)
1552 struct trace_event_call
*call
= event_file_data(m
->private);
1553 struct ftrace_event_field
*field
;
1554 const char *array_descriptor
;
1556 switch ((unsigned long)v
) {
1558 seq_printf(m
, "name: %s\n", trace_event_name(call
));
1559 seq_printf(m
, "ID: %d\n", call
->event
.type
);
1560 seq_puts(m
, "format:\n");
1563 case FORMAT_FIELD_SEPERATOR
:
1567 case FORMAT_PRINTFMT
:
1568 seq_printf(m
, "\nprint fmt: %s\n",
1573 field
= list_entry(v
, struct ftrace_event_field
, link
);
1575 * Smartly shows the array type(except dynamic array).
1578 * If TYPE := TYPE[LEN], it is shown:
1579 * field:TYPE VAR[LEN]
1581 array_descriptor
= strchr(field
->type
, '[');
1583 if (str_has_prefix(field
->type
, "__data_loc"))
1584 array_descriptor
= NULL
;
1586 if (!array_descriptor
)
1587 seq_printf(m
, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1588 field
->type
, field
->name
, field
->offset
,
1589 field
->size
, !!field
->is_signed
);
1591 seq_printf(m
, "\tfield:%.*s %s%s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1592 (int)(array_descriptor
- field
->type
),
1593 field
->type
, field
->name
,
1594 array_descriptor
, field
->offset
,
1595 field
->size
, !!field
->is_signed
);
1600 static void *f_start(struct seq_file
*m
, loff_t
*pos
)
1602 void *p
= (void *)FORMAT_HEADER
;
1605 /* ->stop() is called even if ->start() fails */
1606 mutex_lock(&event_mutex
);
1607 if (!event_file_data(m
->private))
1608 return ERR_PTR(-ENODEV
);
1610 while (l
< *pos
&& p
)
1611 p
= f_next(m
, p
, &l
);
1616 static void f_stop(struct seq_file
*m
, void *p
)
1618 mutex_unlock(&event_mutex
);
1621 static const struct seq_operations trace_format_seq_ops
= {
1628 static int trace_format_open(struct inode
*inode
, struct file
*file
)
1633 /* Do we want to hide event format files on tracefs lockdown? */
1635 ret
= seq_open(file
, &trace_format_seq_ops
);
1639 m
= file
->private_data
;
1646 event_id_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
, loff_t
*ppos
)
1648 int id
= (long)event_file_data(filp
);
1655 len
= sprintf(buf
, "%d\n", id
);
1657 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, len
);
1661 event_filter_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
1664 struct trace_event_file
*file
;
1665 struct trace_seq
*s
;
1671 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
1678 mutex_lock(&event_mutex
);
1679 file
= event_file_data(filp
);
1681 print_event_filter(file
, s
);
1682 mutex_unlock(&event_mutex
);
1685 r
= simple_read_from_buffer(ubuf
, cnt
, ppos
,
1686 s
->buffer
, trace_seq_used(s
));
1694 event_filter_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
1697 struct trace_event_file
*file
;
1701 if (cnt
>= PAGE_SIZE
)
1704 buf
= memdup_user_nul(ubuf
, cnt
);
1706 return PTR_ERR(buf
);
1708 mutex_lock(&event_mutex
);
1709 file
= event_file_data(filp
);
1711 err
= apply_event_filter(file
, buf
);
1712 mutex_unlock(&event_mutex
);
1723 static LIST_HEAD(event_subsystems
);
1725 static int subsystem_open(struct inode
*inode
, struct file
*filp
)
1727 struct trace_subsystem_dir
*dir
= NULL
, *iter_dir
;
1728 struct trace_array
*tr
= NULL
, *iter_tr
;
1729 struct event_subsystem
*system
= NULL
;
1732 if (tracing_is_disabled())
1735 /* Make sure the system still exists */
1736 mutex_lock(&event_mutex
);
1737 mutex_lock(&trace_types_lock
);
1738 list_for_each_entry(iter_tr
, &ftrace_trace_arrays
, list
) {
1739 list_for_each_entry(iter_dir
, &iter_tr
->systems
, list
) {
1740 if (iter_dir
== inode
->i_private
) {
1741 /* Don't open systems with no events */
1744 if (dir
->nr_events
) {
1745 __get_system_dir(dir
);
1746 system
= dir
->subsystem
;
1753 mutex_unlock(&trace_types_lock
);
1754 mutex_unlock(&event_mutex
);
1759 /* Still need to increment the ref count of the system */
1760 if (trace_array_get(tr
) < 0) {
1765 ret
= tracing_open_generic(inode
, filp
);
1767 trace_array_put(tr
);
1774 static int system_tr_open(struct inode
*inode
, struct file
*filp
)
1776 struct trace_subsystem_dir
*dir
;
1777 struct trace_array
*tr
= inode
->i_private
;
1780 /* Make a temporary dir that has no system but points to tr */
1781 dir
= kzalloc(sizeof(*dir
), GFP_KERNEL
);
1785 ret
= tracing_open_generic_tr(inode
, filp
);
1791 filp
->private_data
= dir
;
1796 static int subsystem_release(struct inode
*inode
, struct file
*file
)
1798 struct trace_subsystem_dir
*dir
= file
->private_data
;
1800 trace_array_put(dir
->tr
);
1803 * If dir->subsystem is NULL, then this is a temporary
1804 * descriptor that was made for a trace_array to enable
1816 subsystem_filter_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
1819 struct trace_subsystem_dir
*dir
= filp
->private_data
;
1820 struct event_subsystem
*system
= dir
->subsystem
;
1821 struct trace_seq
*s
;
1827 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
1833 print_subsystem_event_filter(system
, s
);
1834 r
= simple_read_from_buffer(ubuf
, cnt
, ppos
,
1835 s
->buffer
, trace_seq_used(s
));
1843 subsystem_filter_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
1846 struct trace_subsystem_dir
*dir
= filp
->private_data
;
1850 if (cnt
>= PAGE_SIZE
)
1853 buf
= memdup_user_nul(ubuf
, cnt
);
1855 return PTR_ERR(buf
);
1857 err
= apply_subsystem_event_filter(dir
, buf
);
1868 show_header(struct file
*filp
, char __user
*ubuf
, size_t cnt
, loff_t
*ppos
)
1870 int (*func
)(struct trace_seq
*s
) = filp
->private_data
;
1871 struct trace_seq
*s
;
1877 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
1884 r
= simple_read_from_buffer(ubuf
, cnt
, ppos
,
1885 s
->buffer
, trace_seq_used(s
));
1892 static void ignore_task_cpu(void *data
)
1894 struct trace_array
*tr
= data
;
1895 struct trace_pid_list
*pid_list
;
1896 struct trace_pid_list
*no_pid_list
;
1899 * This function is called by on_each_cpu() while the
1900 * event_mutex is held.
1902 pid_list
= rcu_dereference_protected(tr
->filtered_pids
,
1903 mutex_is_locked(&event_mutex
));
1904 no_pid_list
= rcu_dereference_protected(tr
->filtered_no_pids
,
1905 mutex_is_locked(&event_mutex
));
1907 this_cpu_write(tr
->array_buffer
.data
->ignore_pid
,
1908 trace_ignore_this_task(pid_list
, no_pid_list
, current
));
1911 static void register_pid_events(struct trace_array
*tr
)
1914 * Register a probe that is called before all other probes
1915 * to set ignore_pid if next or prev do not match.
1916 * Register a probe this is called after all other probes
1917 * to only keep ignore_pid set if next pid matches.
1919 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre
,
1921 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post
,
1924 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre
,
1926 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post
,
1929 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre
,
1931 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post
,
1934 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre
,
1936 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post
,
1941 event_pid_write(struct file
*filp
, const char __user
*ubuf
,
1942 size_t cnt
, loff_t
*ppos
, int type
)
1944 struct seq_file
*m
= filp
->private_data
;
1945 struct trace_array
*tr
= m
->private;
1946 struct trace_pid_list
*filtered_pids
= NULL
;
1947 struct trace_pid_list
*other_pids
= NULL
;
1948 struct trace_pid_list
*pid_list
;
1949 struct trace_event_file
*file
;
1955 ret
= tracing_update_buffers();
1959 mutex_lock(&event_mutex
);
1961 if (type
== TRACE_PIDS
) {
1962 filtered_pids
= rcu_dereference_protected(tr
->filtered_pids
,
1963 lockdep_is_held(&event_mutex
));
1964 other_pids
= rcu_dereference_protected(tr
->filtered_no_pids
,
1965 lockdep_is_held(&event_mutex
));
1967 filtered_pids
= rcu_dereference_protected(tr
->filtered_no_pids
,
1968 lockdep_is_held(&event_mutex
));
1969 other_pids
= rcu_dereference_protected(tr
->filtered_pids
,
1970 lockdep_is_held(&event_mutex
));
1973 ret
= trace_pid_write(filtered_pids
, &pid_list
, ubuf
, cnt
);
1977 if (type
== TRACE_PIDS
)
1978 rcu_assign_pointer(tr
->filtered_pids
, pid_list
);
1980 rcu_assign_pointer(tr
->filtered_no_pids
, pid_list
);
1982 list_for_each_entry(file
, &tr
->events
, list
) {
1983 set_bit(EVENT_FILE_FL_PID_FILTER_BIT
, &file
->flags
);
1986 if (filtered_pids
) {
1987 tracepoint_synchronize_unregister();
1988 trace_pid_list_free(filtered_pids
);
1989 } else if (pid_list
&& !other_pids
) {
1990 register_pid_events(tr
);
1994 * Ignoring of pids is done at task switch. But we have to
1995 * check for those tasks that are currently running.
1996 * Always do this in case a pid was appended or removed.
1998 on_each_cpu(ignore_task_cpu
, tr
, 1);
2001 mutex_unlock(&event_mutex
);
2010 ftrace_event_pid_write(struct file
*filp
, const char __user
*ubuf
,
2011 size_t cnt
, loff_t
*ppos
)
2013 return event_pid_write(filp
, ubuf
, cnt
, ppos
, TRACE_PIDS
);
2017 ftrace_event_npid_write(struct file
*filp
, const char __user
*ubuf
,
2018 size_t cnt
, loff_t
*ppos
)
2020 return event_pid_write(filp
, ubuf
, cnt
, ppos
, TRACE_NO_PIDS
);
2023 static int ftrace_event_avail_open(struct inode
*inode
, struct file
*file
);
2024 static int ftrace_event_set_open(struct inode
*inode
, struct file
*file
);
2025 static int ftrace_event_set_pid_open(struct inode
*inode
, struct file
*file
);
2026 static int ftrace_event_set_npid_open(struct inode
*inode
, struct file
*file
);
2027 static int ftrace_event_release(struct inode
*inode
, struct file
*file
);
2029 static const struct seq_operations show_event_seq_ops
= {
2036 static const struct seq_operations show_set_event_seq_ops
= {
2043 static const struct seq_operations show_set_pid_seq_ops
= {
2046 .show
= trace_pid_show
,
2050 static const struct seq_operations show_set_no_pid_seq_ops
= {
2053 .show
= trace_pid_show
,
2057 static const struct file_operations ftrace_avail_fops
= {
2058 .open
= ftrace_event_avail_open
,
2060 .llseek
= seq_lseek
,
2061 .release
= seq_release
,
2064 static const struct file_operations ftrace_set_event_fops
= {
2065 .open
= ftrace_event_set_open
,
2067 .write
= ftrace_event_write
,
2068 .llseek
= seq_lseek
,
2069 .release
= ftrace_event_release
,
2072 static const struct file_operations ftrace_set_event_pid_fops
= {
2073 .open
= ftrace_event_set_pid_open
,
2075 .write
= ftrace_event_pid_write
,
2076 .llseek
= seq_lseek
,
2077 .release
= ftrace_event_release
,
2080 static const struct file_operations ftrace_set_event_notrace_pid_fops
= {
2081 .open
= ftrace_event_set_npid_open
,
2083 .write
= ftrace_event_npid_write
,
2084 .llseek
= seq_lseek
,
2085 .release
= ftrace_event_release
,
2088 static const struct file_operations ftrace_enable_fops
= {
2089 .open
= tracing_open_generic
,
2090 .read
= event_enable_read
,
2091 .write
= event_enable_write
,
2092 .llseek
= default_llseek
,
2095 static const struct file_operations ftrace_event_format_fops
= {
2096 .open
= trace_format_open
,
2098 .llseek
= seq_lseek
,
2099 .release
= seq_release
,
2102 static const struct file_operations ftrace_event_id_fops
= {
2103 .read
= event_id_read
,
2104 .llseek
= default_llseek
,
2107 static const struct file_operations ftrace_event_filter_fops
= {
2108 .open
= tracing_open_generic
,
2109 .read
= event_filter_read
,
2110 .write
= event_filter_write
,
2111 .llseek
= default_llseek
,
2114 static const struct file_operations ftrace_subsystem_filter_fops
= {
2115 .open
= subsystem_open
,
2116 .read
= subsystem_filter_read
,
2117 .write
= subsystem_filter_write
,
2118 .llseek
= default_llseek
,
2119 .release
= subsystem_release
,
2122 static const struct file_operations ftrace_system_enable_fops
= {
2123 .open
= subsystem_open
,
2124 .read
= system_enable_read
,
2125 .write
= system_enable_write
,
2126 .llseek
= default_llseek
,
2127 .release
= subsystem_release
,
2130 static const struct file_operations ftrace_tr_enable_fops
= {
2131 .open
= system_tr_open
,
2132 .read
= system_enable_read
,
2133 .write
= system_enable_write
,
2134 .llseek
= default_llseek
,
2135 .release
= subsystem_release
,
2138 static const struct file_operations ftrace_show_header_fops
= {
2139 .open
= tracing_open_generic
,
2140 .read
= show_header
,
2141 .llseek
= default_llseek
,
2145 ftrace_event_open(struct inode
*inode
, struct file
*file
,
2146 const struct seq_operations
*seq_ops
)
2151 ret
= security_locked_down(LOCKDOWN_TRACEFS
);
2155 ret
= seq_open(file
, seq_ops
);
2158 m
= file
->private_data
;
2159 /* copy tr over to seq ops */
2160 m
->private = inode
->i_private
;
2165 static int ftrace_event_release(struct inode
*inode
, struct file
*file
)
2167 struct trace_array
*tr
= inode
->i_private
;
2169 trace_array_put(tr
);
2171 return seq_release(inode
, file
);
2175 ftrace_event_avail_open(struct inode
*inode
, struct file
*file
)
2177 const struct seq_operations
*seq_ops
= &show_event_seq_ops
;
2179 /* Checks for tracefs lockdown */
2180 return ftrace_event_open(inode
, file
, seq_ops
);
2184 ftrace_event_set_open(struct inode
*inode
, struct file
*file
)
2186 const struct seq_operations
*seq_ops
= &show_set_event_seq_ops
;
2187 struct trace_array
*tr
= inode
->i_private
;
2190 ret
= tracing_check_open_get_tr(tr
);
2194 if ((file
->f_mode
& FMODE_WRITE
) &&
2195 (file
->f_flags
& O_TRUNC
))
2196 ftrace_clear_events(tr
);
2198 ret
= ftrace_event_open(inode
, file
, seq_ops
);
2200 trace_array_put(tr
);
2205 ftrace_event_set_pid_open(struct inode
*inode
, struct file
*file
)
2207 const struct seq_operations
*seq_ops
= &show_set_pid_seq_ops
;
2208 struct trace_array
*tr
= inode
->i_private
;
2211 ret
= tracing_check_open_get_tr(tr
);
2215 if ((file
->f_mode
& FMODE_WRITE
) &&
2216 (file
->f_flags
& O_TRUNC
))
2217 ftrace_clear_event_pids(tr
, TRACE_PIDS
);
2219 ret
= ftrace_event_open(inode
, file
, seq_ops
);
2221 trace_array_put(tr
);
2226 ftrace_event_set_npid_open(struct inode
*inode
, struct file
*file
)
2228 const struct seq_operations
*seq_ops
= &show_set_no_pid_seq_ops
;
2229 struct trace_array
*tr
= inode
->i_private
;
2232 ret
= tracing_check_open_get_tr(tr
);
2236 if ((file
->f_mode
& FMODE_WRITE
) &&
2237 (file
->f_flags
& O_TRUNC
))
2238 ftrace_clear_event_pids(tr
, TRACE_NO_PIDS
);
2240 ret
= ftrace_event_open(inode
, file
, seq_ops
);
2242 trace_array_put(tr
);
2246 static struct event_subsystem
*
2247 create_new_subsystem(const char *name
)
2249 struct event_subsystem
*system
;
2251 /* need to create new entry */
2252 system
= kmalloc(sizeof(*system
), GFP_KERNEL
);
2256 system
->ref_count
= 1;
2258 /* Only allocate if dynamic (kprobes and modules) */
2259 system
->name
= kstrdup_const(name
, GFP_KERNEL
);
2263 system
->filter
= NULL
;
2265 system
->filter
= kzalloc(sizeof(struct event_filter
), GFP_KERNEL
);
2266 if (!system
->filter
)
2269 list_add(&system
->list
, &event_subsystems
);
2274 kfree_const(system
->name
);
2279 static struct dentry
*
2280 event_subsystem_dir(struct trace_array
*tr
, const char *name
,
2281 struct trace_event_file
*file
, struct dentry
*parent
)
2283 struct event_subsystem
*system
, *iter
;
2284 struct trace_subsystem_dir
*dir
;
2285 struct dentry
*entry
;
2287 /* First see if we did not already create this dir */
2288 list_for_each_entry(dir
, &tr
->systems
, list
) {
2289 system
= dir
->subsystem
;
2290 if (strcmp(system
->name
, name
) == 0) {
2297 /* Now see if the system itself exists. */
2299 list_for_each_entry(iter
, &event_subsystems
, list
) {
2300 if (strcmp(iter
->name
, name
) == 0) {
2306 dir
= kmalloc(sizeof(*dir
), GFP_KERNEL
);
2311 system
= create_new_subsystem(name
);
2315 __get_system(system
);
2317 dir
->entry
= tracefs_create_dir(name
, parent
);
2319 pr_warn("Failed to create system directory %s\n", name
);
2320 __put_system(system
);
2327 dir
->subsystem
= system
;
2330 /* the ftrace system is special, do not create enable or filter files */
2331 if (strcmp(name
, "ftrace") != 0) {
2333 entry
= tracefs_create_file("filter", TRACE_MODE_WRITE
,
2335 &ftrace_subsystem_filter_fops
);
2337 kfree(system
->filter
);
2338 system
->filter
= NULL
;
2339 pr_warn("Could not create tracefs '%s/filter' entry\n", name
);
2342 trace_create_file("enable", TRACE_MODE_WRITE
, dir
->entry
, dir
,
2343 &ftrace_system_enable_fops
);
2346 list_add(&dir
->list
, &tr
->systems
);
2353 /* Only print this message if failed on memory allocation */
2354 if (!dir
|| !system
)
2355 pr_warn("No memory to create event subsystem %s\n", name
);
2360 event_define_fields(struct trace_event_call
*call
)
2362 struct list_head
*head
;
2366 * Other events may have the same class. Only update
2367 * the fields if they are not already defined.
2369 head
= trace_get_fields(call
);
2370 if (list_empty(head
)) {
2371 struct trace_event_fields
*field
= call
->class->fields_array
;
2372 unsigned int offset
= sizeof(struct trace_entry
);
2374 for (; field
->type
; field
++) {
2375 if (field
->type
== TRACE_FUNCTION_TYPE
) {
2376 field
->define_fields(call
);
2380 offset
= ALIGN(offset
, field
->align
);
2381 ret
= trace_define_field(call
, field
->type
, field
->name
,
2382 offset
, field
->size
,
2383 field
->is_signed
, field
->filter_type
);
2384 if (WARN_ON_ONCE(ret
)) {
2385 pr_err("error code is %d\n", ret
);
2389 offset
+= field
->size
;
2397 event_create_dir(struct dentry
*parent
, struct trace_event_file
*file
)
2399 struct trace_event_call
*call
= file
->event_call
;
2400 struct trace_array
*tr
= file
->tr
;
2401 struct dentry
*d_events
;
2406 * If the trace point header did not define TRACE_SYSTEM
2407 * then the system would be called "TRACE_SYSTEM".
2409 if (strcmp(call
->class->system
, TRACE_SYSTEM
) != 0) {
2410 d_events
= event_subsystem_dir(tr
, call
->class->system
, file
, parent
);
2416 name
= trace_event_name(call
);
2417 file
->dir
= tracefs_create_dir(name
, d_events
);
2419 pr_warn("Could not create tracefs '%s' directory\n", name
);
2423 if (call
->class->reg
&& !(call
->flags
& TRACE_EVENT_FL_IGNORE_ENABLE
))
2424 trace_create_file("enable", TRACE_MODE_WRITE
, file
->dir
, file
,
2425 &ftrace_enable_fops
);
2427 #ifdef CONFIG_PERF_EVENTS
2428 if (call
->event
.type
&& call
->class->reg
)
2429 trace_create_file("id", TRACE_MODE_READ
, file
->dir
,
2430 (void *)(long)call
->event
.type
,
2431 &ftrace_event_id_fops
);
2434 ret
= event_define_fields(call
);
2436 pr_warn("Could not initialize trace point events/%s\n", name
);
2441 * Only event directories that can be enabled should have
2442 * triggers or filters.
2444 if (!(call
->flags
& TRACE_EVENT_FL_IGNORE_ENABLE
)) {
2445 trace_create_file("filter", TRACE_MODE_WRITE
, file
->dir
,
2446 file
, &ftrace_event_filter_fops
);
2448 trace_create_file("trigger", TRACE_MODE_WRITE
, file
->dir
,
2449 file
, &event_trigger_fops
);
2452 #ifdef CONFIG_HIST_TRIGGERS
2453 trace_create_file("hist", TRACE_MODE_READ
, file
->dir
, file
,
2456 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2457 trace_create_file("hist_debug", TRACE_MODE_READ
, file
->dir
, file
,
2458 &event_hist_debug_fops
);
2460 trace_create_file("format", TRACE_MODE_READ
, file
->dir
, call
,
2461 &ftrace_event_format_fops
);
2463 #ifdef CONFIG_TRACE_EVENT_INJECT
2464 if (call
->event
.type
&& call
->class->reg
)
2465 trace_create_file("inject", 0200, file
->dir
, file
,
2466 &event_inject_fops
);
2472 static void remove_event_from_tracers(struct trace_event_call
*call
)
2474 struct trace_event_file
*file
;
2475 struct trace_array
*tr
;
2477 do_for_each_event_file_safe(tr
, file
) {
2478 if (file
->event_call
!= call
)
2481 remove_event_file_dir(file
);
2483 * The do_for_each_event_file_safe() is
2484 * a double loop. After finding the call for this
2485 * trace_array, we use break to jump to the next
2489 } while_for_each_event_file();
2492 static void event_remove(struct trace_event_call
*call
)
2494 struct trace_array
*tr
;
2495 struct trace_event_file
*file
;
2497 do_for_each_event_file(tr
, file
) {
2498 if (file
->event_call
!= call
)
2501 if (file
->flags
& EVENT_FILE_FL_WAS_ENABLED
)
2502 tr
->clear_trace
= true;
2504 ftrace_event_enable_disable(file
, 0);
2506 * The do_for_each_event_file() is
2507 * a double loop. After finding the call for this
2508 * trace_array, we use break to jump to the next
2512 } while_for_each_event_file();
2514 if (call
->event
.funcs
)
2515 __unregister_trace_event(&call
->event
);
2516 remove_event_from_tracers(call
);
2517 list_del(&call
->list
);
2520 static int event_init(struct trace_event_call
*call
)
2525 name
= trace_event_name(call
);
2529 if (call
->class->raw_init
) {
2530 ret
= call
->class->raw_init(call
);
2531 if (ret
< 0 && ret
!= -ENOSYS
)
2532 pr_warn("Could not initialize trace events/%s\n", name
);
2539 __register_event(struct trace_event_call
*call
, struct module
*mod
)
2543 ret
= event_init(call
);
2547 list_add(&call
->list
, &ftrace_events
);
2548 if (call
->flags
& TRACE_EVENT_FL_DYNAMIC
)
2549 atomic_set(&call
->refcnt
, 0);
2556 static char *eval_replace(char *ptr
, struct trace_eval_map
*map
, int len
)
2561 /* Find the length of the eval value as a string */
2562 elen
= snprintf(ptr
, 0, "%ld", map
->eval_value
);
2563 /* Make sure there's enough room to replace the string with the value */
2567 snprintf(ptr
, elen
+ 1, "%ld", map
->eval_value
);
2569 /* Get the rest of the string of ptr */
2570 rlen
= strlen(ptr
+ len
);
2571 memmove(ptr
+ elen
, ptr
+ len
, rlen
);
2572 /* Make sure we end the new string */
2573 ptr
[elen
+ rlen
] = 0;
2578 static void update_event_printk(struct trace_event_call
*call
,
2579 struct trace_eval_map
*map
)
2583 int len
= strlen(map
->eval_string
);
2585 for (ptr
= call
->print_fmt
; *ptr
; ptr
++) {
2599 if (isdigit(*ptr
)) {
2603 /* Check for alpha chars like ULL */
2604 } while (isalnum(*ptr
));
2608 * A number must have some kind of delimiter after
2609 * it, and we can ignore that too.
2613 if (isalpha(*ptr
) || *ptr
== '_') {
2614 if (strncmp(map
->eval_string
, ptr
, len
) == 0 &&
2615 !isalnum(ptr
[len
]) && ptr
[len
] != '_') {
2616 ptr
= eval_replace(ptr
, map
, len
);
2617 /* enum/sizeof string smaller than value */
2618 if (WARN_ON_ONCE(!ptr
))
2621 * No need to decrement here, as eval_replace()
2622 * returns the pointer to the character passed
2623 * the eval, and two evals can not be placed
2624 * back to back without something in between.
2625 * We can skip that something in between.
2632 } while (isalnum(*ptr
) || *ptr
== '_');
2636 * If what comes after this variable is a '.' or
2637 * '->' then we can continue to ignore that string.
2639 if (*ptr
== '.' || (ptr
[0] == '-' && ptr
[1] == '>')) {
2640 ptr
+= *ptr
== '.' ? 1 : 2;
2646 * Once again, we can skip the delimiter that came
2654 static void add_str_to_module(struct module
*module
, char *str
)
2656 struct module_string
*modstr
;
2658 modstr
= kmalloc(sizeof(*modstr
), GFP_KERNEL
);
2661 * If we failed to allocate memory here, then we'll just
2662 * let the str memory leak when the module is removed.
2663 * If this fails to allocate, there's worse problems than
2664 * a leaked string on module removal.
2666 if (WARN_ON_ONCE(!modstr
))
2669 modstr
->module
= module
;
2672 list_add(&modstr
->next
, &module_strings
);
2675 static void update_event_fields(struct trace_event_call
*call
,
2676 struct trace_eval_map
*map
)
2678 struct ftrace_event_field
*field
;
2679 struct list_head
*head
;
2682 int len
= strlen(map
->eval_string
);
2684 /* Dynamic events should never have field maps */
2685 if (WARN_ON_ONCE(call
->flags
& TRACE_EVENT_FL_DYNAMIC
))
2688 head
= trace_get_fields(call
);
2689 list_for_each_entry(field
, head
, link
) {
2690 ptr
= strchr(field
->type
, '[');
2695 if (!isalpha(*ptr
) && *ptr
!= '_')
2698 if (strncmp(map
->eval_string
, ptr
, len
) != 0)
2701 str
= kstrdup(field
->type
, GFP_KERNEL
);
2702 if (WARN_ON_ONCE(!str
))
2704 ptr
= str
+ (ptr
- field
->type
);
2705 ptr
= eval_replace(ptr
, map
, len
);
2706 /* enum/sizeof string smaller than value */
2707 if (WARN_ON_ONCE(!ptr
)) {
2713 * If the event is part of a module, then we need to free the string
2714 * when the module is removed. Otherwise, it will stay allocated
2718 add_str_to_module(call
->module
, str
);
2724 void trace_event_eval_update(struct trace_eval_map
**map
, int len
)
2726 struct trace_event_call
*call
, *p
;
2727 const char *last_system
= NULL
;
2732 down_write(&trace_event_sem
);
2733 list_for_each_entry_safe(call
, p
, &ftrace_events
, list
) {
2734 /* events are usually grouped together with systems */
2735 if (!last_system
|| call
->class->system
!= last_system
) {
2738 last_system
= call
->class->system
;
2742 * Since calls are grouped by systems, the likelihood that the
2743 * next call in the iteration belongs to the same system as the
2744 * previous call is high. As an optimization, we skip searching
2745 * for a map[] that matches the call's system if the last call
2746 * was from the same system. That's what last_i is for. If the
2747 * call has the same system as the previous call, then last_i
2748 * will be the index of the first map[] that has a matching
2751 for (i
= last_i
; i
< len
; i
++) {
2752 if (call
->class->system
== map
[i
]->system
) {
2753 /* Save the first system if need be */
2758 update_event_printk(call
, map
[i
]);
2759 update_event_fields(call
, map
[i
]);
2763 up_write(&trace_event_sem
);
2766 static struct trace_event_file
*
2767 trace_create_new_event(struct trace_event_call
*call
,
2768 struct trace_array
*tr
)
2770 struct trace_pid_list
*no_pid_list
;
2771 struct trace_pid_list
*pid_list
;
2772 struct trace_event_file
*file
;
2775 file
= kmem_cache_alloc(file_cachep
, GFP_TRACE
);
2779 pid_list
= rcu_dereference_protected(tr
->filtered_pids
,
2780 lockdep_is_held(&event_mutex
));
2781 no_pid_list
= rcu_dereference_protected(tr
->filtered_no_pids
,
2782 lockdep_is_held(&event_mutex
));
2784 if (!trace_pid_list_first(pid_list
, &first
) ||
2785 !trace_pid_list_first(no_pid_list
, &first
))
2786 file
->flags
|= EVENT_FILE_FL_PID_FILTER
;
2788 file
->event_call
= call
;
2790 atomic_set(&file
->sm_ref
, 0);
2791 atomic_set(&file
->tm_ref
, 0);
2792 INIT_LIST_HEAD(&file
->triggers
);
2793 list_add(&file
->list
, &tr
->events
);
2798 /* Add an event to a trace directory */
2800 __trace_add_new_event(struct trace_event_call
*call
, struct trace_array
*tr
)
2802 struct trace_event_file
*file
;
2804 file
= trace_create_new_event(call
, tr
);
2808 if (eventdir_initialized
)
2809 return event_create_dir(tr
->event_dir
, file
);
2811 return event_define_fields(call
);
2815 * Just create a descriptor for early init. A descriptor is required
2816 * for enabling events at boot. We want to enable events before
2817 * the filesystem is initialized.
2820 __trace_early_add_new_event(struct trace_event_call
*call
,
2821 struct trace_array
*tr
)
2823 struct trace_event_file
*file
;
2825 file
= trace_create_new_event(call
, tr
);
2829 return event_define_fields(call
);
2832 struct ftrace_module_file_ops
;
2833 static void __add_event_to_tracers(struct trace_event_call
*call
);
2835 /* Add an additional event_call dynamically */
2836 int trace_add_event_call(struct trace_event_call
*call
)
2839 lockdep_assert_held(&event_mutex
);
2841 mutex_lock(&trace_types_lock
);
2843 ret
= __register_event(call
, NULL
);
2845 __add_event_to_tracers(call
);
2847 mutex_unlock(&trace_types_lock
);
2850 EXPORT_SYMBOL_GPL(trace_add_event_call
);
2853 * Must be called under locking of trace_types_lock, event_mutex and
2856 static void __trace_remove_event_call(struct trace_event_call
*call
)
2859 trace_destroy_fields(call
);
2860 free_event_filter(call
->filter
);
2861 call
->filter
= NULL
;
2864 static int probe_remove_event_call(struct trace_event_call
*call
)
2866 struct trace_array
*tr
;
2867 struct trace_event_file
*file
;
2869 #ifdef CONFIG_PERF_EVENTS
2870 if (call
->perf_refcount
)
2873 do_for_each_event_file(tr
, file
) {
2874 if (file
->event_call
!= call
)
2877 * We can't rely on ftrace_event_enable_disable(enable => 0)
2878 * we are going to do, EVENT_FILE_FL_SOFT_MODE can suppress
2879 * TRACE_REG_UNREGISTER.
2881 if (file
->flags
& EVENT_FILE_FL_ENABLED
)
2884 * The do_for_each_event_file_safe() is
2885 * a double loop. After finding the call for this
2886 * trace_array, we use break to jump to the next
2890 } while_for_each_event_file();
2892 __trace_remove_event_call(call
);
2897 /* Remove an event_call */
2898 int trace_remove_event_call(struct trace_event_call
*call
)
2902 lockdep_assert_held(&event_mutex
);
2904 mutex_lock(&trace_types_lock
);
2905 down_write(&trace_event_sem
);
2906 ret
= probe_remove_event_call(call
);
2907 up_write(&trace_event_sem
);
2908 mutex_unlock(&trace_types_lock
);
2912 EXPORT_SYMBOL_GPL(trace_remove_event_call
);
2914 #define for_each_event(event, start, end) \
2915 for (event = start; \
2916 (unsigned long)event < (unsigned long)end; \
2919 #ifdef CONFIG_MODULES
2921 static void trace_module_add_events(struct module
*mod
)
2923 struct trace_event_call
**call
, **start
, **end
;
2925 if (!mod
->num_trace_events
)
2928 /* Don't add infrastructure for mods without tracepoints */
2929 if (trace_module_has_bad_taint(mod
)) {
2930 pr_err("%s: module has bad taint, not creating trace events\n",
2935 start
= mod
->trace_events
;
2936 end
= mod
->trace_events
+ mod
->num_trace_events
;
2938 for_each_event(call
, start
, end
) {
2939 __register_event(*call
, mod
);
2940 __add_event_to_tracers(*call
);
2944 static void trace_module_remove_events(struct module
*mod
)
2946 struct trace_event_call
*call
, *p
;
2947 struct module_string
*modstr
, *m
;
2949 down_write(&trace_event_sem
);
2950 list_for_each_entry_safe(call
, p
, &ftrace_events
, list
) {
2951 if ((call
->flags
& TRACE_EVENT_FL_DYNAMIC
) || !call
->module
)
2953 if (call
->module
== mod
)
2954 __trace_remove_event_call(call
);
2956 /* Check for any strings allocade for this module */
2957 list_for_each_entry_safe(modstr
, m
, &module_strings
, next
) {
2958 if (modstr
->module
!= mod
)
2960 list_del(&modstr
->next
);
2964 up_write(&trace_event_sem
);
2967 * It is safest to reset the ring buffer if the module being unloaded
2968 * registered any events that were used. The only worry is if
2969 * a new module gets loaded, and takes on the same id as the events
2970 * of this module. When printing out the buffer, traced events left
2971 * over from this module may be passed to the new module events and
2972 * unexpected results may occur.
2974 tracing_reset_all_online_cpus();
2977 static int trace_module_notify(struct notifier_block
*self
,
2978 unsigned long val
, void *data
)
2980 struct module
*mod
= data
;
2982 mutex_lock(&event_mutex
);
2983 mutex_lock(&trace_types_lock
);
2985 case MODULE_STATE_COMING
:
2986 trace_module_add_events(mod
);
2988 case MODULE_STATE_GOING
:
2989 trace_module_remove_events(mod
);
2992 mutex_unlock(&trace_types_lock
);
2993 mutex_unlock(&event_mutex
);
2998 static struct notifier_block trace_module_nb
= {
2999 .notifier_call
= trace_module_notify
,
3000 .priority
= 1, /* higher than trace.c module notify */
3002 #endif /* CONFIG_MODULES */
3004 /* Create a new event directory structure for a trace directory. */
3006 __trace_add_event_dirs(struct trace_array
*tr
)
3008 struct trace_event_call
*call
;
3011 list_for_each_entry(call
, &ftrace_events
, list
) {
3012 ret
= __trace_add_new_event(call
, tr
);
3014 pr_warn("Could not create directory for event %s\n",
3015 trace_event_name(call
));
3019 /* Returns any file that matches the system and event */
3020 struct trace_event_file
*
3021 __find_event_file(struct trace_array
*tr
, const char *system
, const char *event
)
3023 struct trace_event_file
*file
;
3024 struct trace_event_call
*call
;
3027 list_for_each_entry(file
, &tr
->events
, list
) {
3029 call
= file
->event_call
;
3030 name
= trace_event_name(call
);
3032 if (!name
|| !call
->class)
3035 if (strcmp(event
, name
) == 0 &&
3036 strcmp(system
, call
->class->system
) == 0)
3042 /* Returns valid trace event files that match system and event */
3043 struct trace_event_file
*
3044 find_event_file(struct trace_array
*tr
, const char *system
, const char *event
)
3046 struct trace_event_file
*file
;
3048 file
= __find_event_file(tr
, system
, event
);
3049 if (!file
|| !file
->event_call
->class->reg
||
3050 file
->event_call
->flags
& TRACE_EVENT_FL_IGNORE_ENABLE
)
3057 * trace_get_event_file - Find and return a trace event file
3058 * @instance: The name of the trace instance containing the event
3059 * @system: The name of the system containing the event
3060 * @event: The name of the event
3062 * Return a trace event file given the trace instance name, trace
3063 * system, and trace event name. If the instance name is NULL, it
3064 * refers to the top-level trace array.
3066 * This function will look it up and return it if found, after calling
3067 * trace_array_get() to prevent the instance from going away, and
3068 * increment the event's module refcount to prevent it from being
3071 * To release the file, call trace_put_event_file(), which will call
3072 * trace_array_put() and decrement the event's module refcount.
3074 * Return: The trace event on success, ERR_PTR otherwise.
3076 struct trace_event_file
*trace_get_event_file(const char *instance
,
3080 struct trace_array
*tr
= top_trace_array();
3081 struct trace_event_file
*file
= NULL
;
3085 tr
= trace_array_find_get(instance
);
3087 return ERR_PTR(-ENOENT
);
3089 ret
= trace_array_get(tr
);
3091 return ERR_PTR(ret
);
3094 mutex_lock(&event_mutex
);
3096 file
= find_event_file(tr
, system
, event
);
3098 trace_array_put(tr
);
3103 /* Don't let event modules unload while in use */
3104 ret
= trace_event_try_get_ref(file
->event_call
);
3106 trace_array_put(tr
);
3113 mutex_unlock(&event_mutex
);
3116 file
= ERR_PTR(ret
);
3120 EXPORT_SYMBOL_GPL(trace_get_event_file
);
3123 * trace_put_event_file - Release a file from trace_get_event_file()
3124 * @file: The trace event file
3126 * If a file was retrieved using trace_get_event_file(), this should
3127 * be called when it's no longer needed. It will cancel the previous
3128 * trace_array_get() called by that function, and decrement the
3129 * event's module refcount.
3131 void trace_put_event_file(struct trace_event_file
*file
)
3133 mutex_lock(&event_mutex
);
3134 trace_event_put_ref(file
->event_call
);
3135 mutex_unlock(&event_mutex
);
3137 trace_array_put(file
->tr
);
3139 EXPORT_SYMBOL_GPL(trace_put_event_file
);
3141 #ifdef CONFIG_DYNAMIC_FTRACE
3144 #define ENABLE_EVENT_STR "enable_event"
3145 #define DISABLE_EVENT_STR "disable_event"
3147 struct event_probe_data
{
3148 struct trace_event_file
*file
;
3149 unsigned long count
;
3154 static void update_event_probe(struct event_probe_data
*data
)
3157 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT
, &data
->file
->flags
);
3159 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT
, &data
->file
->flags
);
3163 event_enable_probe(unsigned long ip
, unsigned long parent_ip
,
3164 struct trace_array
*tr
, struct ftrace_probe_ops
*ops
,
3167 struct ftrace_func_mapper
*mapper
= data
;
3168 struct event_probe_data
*edata
;
3171 pdata
= ftrace_func_mapper_find_ip(mapper
, ip
);
3172 if (!pdata
|| !*pdata
)
3176 update_event_probe(edata
);
3180 event_enable_count_probe(unsigned long ip
, unsigned long parent_ip
,
3181 struct trace_array
*tr
, struct ftrace_probe_ops
*ops
,
3184 struct ftrace_func_mapper
*mapper
= data
;
3185 struct event_probe_data
*edata
;
3188 pdata
= ftrace_func_mapper_find_ip(mapper
, ip
);
3189 if (!pdata
|| !*pdata
)
3197 /* Skip if the event is in a state we want to switch to */
3198 if (edata
->enable
== !(edata
->file
->flags
& EVENT_FILE_FL_SOFT_DISABLED
))
3201 if (edata
->count
!= -1)
3204 update_event_probe(edata
);
3208 event_enable_print(struct seq_file
*m
, unsigned long ip
,
3209 struct ftrace_probe_ops
*ops
, void *data
)
3211 struct ftrace_func_mapper
*mapper
= data
;
3212 struct event_probe_data
*edata
;
3215 pdata
= ftrace_func_mapper_find_ip(mapper
, ip
);
3217 if (WARN_ON_ONCE(!pdata
|| !*pdata
))
3222 seq_printf(m
, "%ps:", (void *)ip
);
3224 seq_printf(m
, "%s:%s:%s",
3225 edata
->enable
? ENABLE_EVENT_STR
: DISABLE_EVENT_STR
,
3226 edata
->file
->event_call
->class->system
,
3227 trace_event_name(edata
->file
->event_call
));
3229 if (edata
->count
== -1)
3230 seq_puts(m
, ":unlimited\n");
3232 seq_printf(m
, ":count=%ld\n", edata
->count
);
3238 event_enable_init(struct ftrace_probe_ops
*ops
, struct trace_array
*tr
,
3239 unsigned long ip
, void *init_data
, void **data
)
3241 struct ftrace_func_mapper
*mapper
= *data
;
3242 struct event_probe_data
*edata
= init_data
;
3246 mapper
= allocate_ftrace_func_mapper();
3252 ret
= ftrace_func_mapper_add_ip(mapper
, ip
, edata
);
3261 static int free_probe_data(void *data
)
3263 struct event_probe_data
*edata
= data
;
3267 /* Remove the SOFT_MODE flag */
3268 __ftrace_event_enable_disable(edata
->file
, 0, 1);
3269 trace_event_put_ref(edata
->file
->event_call
);
3276 event_enable_free(struct ftrace_probe_ops
*ops
, struct trace_array
*tr
,
3277 unsigned long ip
, void *data
)
3279 struct ftrace_func_mapper
*mapper
= data
;
3280 struct event_probe_data
*edata
;
3285 free_ftrace_func_mapper(mapper
, free_probe_data
);
3289 edata
= ftrace_func_mapper_remove_ip(mapper
, ip
);
3291 if (WARN_ON_ONCE(!edata
))
3294 if (WARN_ON_ONCE(edata
->ref
<= 0))
3297 free_probe_data(edata
);
3300 static struct ftrace_probe_ops event_enable_probe_ops
= {
3301 .func
= event_enable_probe
,
3302 .print
= event_enable_print
,
3303 .init
= event_enable_init
,
3304 .free
= event_enable_free
,
3307 static struct ftrace_probe_ops event_enable_count_probe_ops
= {
3308 .func
= event_enable_count_probe
,
3309 .print
= event_enable_print
,
3310 .init
= event_enable_init
,
3311 .free
= event_enable_free
,
3314 static struct ftrace_probe_ops event_disable_probe_ops
= {
3315 .func
= event_enable_probe
,
3316 .print
= event_enable_print
,
3317 .init
= event_enable_init
,
3318 .free
= event_enable_free
,
3321 static struct ftrace_probe_ops event_disable_count_probe_ops
= {
3322 .func
= event_enable_count_probe
,
3323 .print
= event_enable_print
,
3324 .init
= event_enable_init
,
3325 .free
= event_enable_free
,
3329 event_enable_func(struct trace_array
*tr
, struct ftrace_hash
*hash
,
3330 char *glob
, char *cmd
, char *param
, int enabled
)
3332 struct trace_event_file
*file
;
3333 struct ftrace_probe_ops
*ops
;
3334 struct event_probe_data
*data
;
3344 /* hash funcs only work with set_ftrace_filter */
3345 if (!enabled
|| !param
)
3348 system
= strsep(¶m
, ":");
3352 event
= strsep(¶m
, ":");
3354 mutex_lock(&event_mutex
);
3357 file
= find_event_file(tr
, system
, event
);
3361 enable
= strcmp(cmd
, ENABLE_EVENT_STR
) == 0;
3364 ops
= param
? &event_enable_count_probe_ops
: &event_enable_probe_ops
;
3366 ops
= param
? &event_disable_count_probe_ops
: &event_disable_probe_ops
;
3368 if (glob
[0] == '!') {
3369 ret
= unregister_ftrace_function_probe_func(glob
+1, tr
, ops
);
3375 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
3379 data
->enable
= enable
;
3386 number
= strsep(¶m
, ":");
3389 if (!strlen(number
))
3393 * We use the callback data field (which is a pointer)
3396 ret
= kstrtoul(number
, 0, &data
->count
);
3401 /* Don't let event modules unload while probe registered */
3402 ret
= trace_event_try_get_ref(file
->event_call
);
3408 ret
= __ftrace_event_enable_disable(file
, 1, 1);
3412 ret
= register_ftrace_function_probe(glob
, tr
, ops
, data
);
3414 * The above returns on success the # of functions enabled,
3415 * but if it didn't find any functions it returns zero.
3416 * Consider no functions a failure too.
3423 /* Just return zero, not the number of enabled functions */
3426 mutex_unlock(&event_mutex
);
3430 __ftrace_event_enable_disable(file
, 0, 1);
3432 trace_event_put_ref(file
->event_call
);
3438 static struct ftrace_func_command event_enable_cmd
= {
3439 .name
= ENABLE_EVENT_STR
,
3440 .func
= event_enable_func
,
3443 static struct ftrace_func_command event_disable_cmd
= {
3444 .name
= DISABLE_EVENT_STR
,
3445 .func
= event_enable_func
,
3448 static __init
int register_event_cmds(void)
3452 ret
= register_ftrace_command(&event_enable_cmd
);
3453 if (WARN_ON(ret
< 0))
3455 ret
= register_ftrace_command(&event_disable_cmd
);
3456 if (WARN_ON(ret
< 0))
3457 unregister_ftrace_command(&event_enable_cmd
);
3461 static inline int register_event_cmds(void) { return 0; }
3462 #endif /* CONFIG_DYNAMIC_FTRACE */
3465 * The top level array and trace arrays created by boot-time tracing
3466 * have already had its trace_event_file descriptors created in order
3467 * to allow for early events to be recorded.
3468 * This function is called after the tracefs has been initialized,
3469 * and we now have to create the files associated to the events.
3471 static void __trace_early_add_event_dirs(struct trace_array
*tr
)
3473 struct trace_event_file
*file
;
3477 list_for_each_entry(file
, &tr
->events
, list
) {
3478 ret
= event_create_dir(tr
->event_dir
, file
);
3480 pr_warn("Could not create directory for event %s\n",
3481 trace_event_name(file
->event_call
));
3486 * For early boot up, the top trace array and the trace arrays created
3487 * by boot-time tracing require to have a list of events that can be
3488 * enabled. This must be done before the filesystem is set up in order
3489 * to allow events to be traced early.
3491 void __trace_early_add_events(struct trace_array
*tr
)
3493 struct trace_event_call
*call
;
3496 list_for_each_entry(call
, &ftrace_events
, list
) {
3497 /* Early boot up should not have any modules loaded */
3498 if (!(call
->flags
& TRACE_EVENT_FL_DYNAMIC
) &&
3499 WARN_ON_ONCE(call
->module
))
3502 ret
= __trace_early_add_new_event(call
, tr
);
3504 pr_warn("Could not create early event %s\n",
3505 trace_event_name(call
));
3509 /* Remove the event directory structure for a trace directory. */
3511 __trace_remove_event_dirs(struct trace_array
*tr
)
3513 struct trace_event_file
*file
, *next
;
3515 list_for_each_entry_safe(file
, next
, &tr
->events
, list
)
3516 remove_event_file_dir(file
);
3519 static void __add_event_to_tracers(struct trace_event_call
*call
)
3521 struct trace_array
*tr
;
3523 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
)
3524 __trace_add_new_event(call
, tr
);
3527 extern struct trace_event_call
*__start_ftrace_events
[];
3528 extern struct trace_event_call
*__stop_ftrace_events
[];
3530 static char bootup_event_buf
[COMMAND_LINE_SIZE
] __initdata
;
3532 static __init
int setup_trace_event(char *str
)
3534 strlcpy(bootup_event_buf
, str
, COMMAND_LINE_SIZE
);
3535 ring_buffer_expanded
= true;
3536 disable_tracing_selftest("running event tracing");
3540 __setup("trace_event=", setup_trace_event
);
3542 /* Expects to have event_mutex held when called */
3544 create_event_toplevel_files(struct dentry
*parent
, struct trace_array
*tr
)
3546 struct dentry
*d_events
;
3547 struct dentry
*entry
;
3549 entry
= trace_create_file("set_event", TRACE_MODE_WRITE
, parent
,
3550 tr
, &ftrace_set_event_fops
);
3554 d_events
= tracefs_create_dir("events", parent
);
3556 pr_warn("Could not create tracefs 'events' directory\n");
3560 entry
= trace_create_file("enable", TRACE_MODE_WRITE
, d_events
,
3561 tr
, &ftrace_tr_enable_fops
);
3565 /* There are not as crucial, just warn if they are not created */
3567 trace_create_file("set_event_pid", TRACE_MODE_WRITE
, parent
,
3568 tr
, &ftrace_set_event_pid_fops
);
3570 trace_create_file("set_event_notrace_pid",
3571 TRACE_MODE_WRITE
, parent
, tr
,
3572 &ftrace_set_event_notrace_pid_fops
);
3574 /* ring buffer internal formats */
3575 trace_create_file("header_page", TRACE_MODE_READ
, d_events
,
3576 ring_buffer_print_page_header
,
3577 &ftrace_show_header_fops
);
3579 trace_create_file("header_event", TRACE_MODE_READ
, d_events
,
3580 ring_buffer_print_entry_header
,
3581 &ftrace_show_header_fops
);
3583 tr
->event_dir
= d_events
;
3589 * event_trace_add_tracer - add a instance of a trace_array to events
3590 * @parent: The parent dentry to place the files/directories for events in
3591 * @tr: The trace array associated with these events
3593 * When a new instance is created, it needs to set up its events
3594 * directory, as well as other files associated with events. It also
3595 * creates the event hierarchy in the @parent/events directory.
3597 * Returns 0 on success.
3599 * Must be called with event_mutex held.
3601 int event_trace_add_tracer(struct dentry
*parent
, struct trace_array
*tr
)
3605 lockdep_assert_held(&event_mutex
);
3607 ret
= create_event_toplevel_files(parent
, tr
);
3611 down_write(&trace_event_sem
);
3612 /* If tr already has the event list, it is initialized in early boot. */
3613 if (unlikely(!list_empty(&tr
->events
)))
3614 __trace_early_add_event_dirs(tr
);
3616 __trace_add_event_dirs(tr
);
3617 up_write(&trace_event_sem
);
3624 * The top trace array already had its file descriptors created.
3625 * Now the files themselves need to be created.
3628 early_event_add_tracer(struct dentry
*parent
, struct trace_array
*tr
)
3632 mutex_lock(&event_mutex
);
3634 ret
= create_event_toplevel_files(parent
, tr
);
3638 down_write(&trace_event_sem
);
3639 __trace_early_add_event_dirs(tr
);
3640 up_write(&trace_event_sem
);
3643 mutex_unlock(&event_mutex
);
3648 /* Must be called with event_mutex held */
3649 int event_trace_del_tracer(struct trace_array
*tr
)
3651 lockdep_assert_held(&event_mutex
);
3653 /* Disable any event triggers and associated soft-disabled events */
3654 clear_event_triggers(tr
);
3656 /* Clear the pid list */
3657 __ftrace_clear_event_pids(tr
, TRACE_PIDS
| TRACE_NO_PIDS
);
3659 /* Disable any running events */
3660 __ftrace_set_clr_event_nolock(tr
, NULL
, NULL
, NULL
, 0);
3662 /* Make sure no more events are being executed */
3663 tracepoint_synchronize_unregister();
3665 down_write(&trace_event_sem
);
3666 __trace_remove_event_dirs(tr
);
3667 tracefs_remove(tr
->event_dir
);
3668 up_write(&trace_event_sem
);
3670 tr
->event_dir
= NULL
;
3675 static __init
int event_trace_memsetup(void)
3677 field_cachep
= KMEM_CACHE(ftrace_event_field
, SLAB_PANIC
);
3678 file_cachep
= KMEM_CACHE(trace_event_file
, SLAB_PANIC
);
3683 early_enable_events(struct trace_array
*tr
, bool disable_first
)
3685 char *buf
= bootup_event_buf
;
3690 token
= strsep(&buf
, ",");
3696 /* Restarting syscalls requires that we stop them first */
3698 ftrace_set_clr_event(tr
, token
, 0);
3700 ret
= ftrace_set_clr_event(tr
, token
, 1);
3702 pr_warn("Failed to enable trace event: %s\n", token
);
3705 /* Put back the comma to allow this to be called again */
3711 static __init
int event_trace_enable(void)
3713 struct trace_array
*tr
= top_trace_array();
3714 struct trace_event_call
**iter
, *call
;
3720 for_each_event(iter
, __start_ftrace_events
, __stop_ftrace_events
) {
3723 ret
= event_init(call
);
3725 list_add(&call
->list
, &ftrace_events
);
3729 * We need the top trace array to have a working set of trace
3730 * points at early init, before the debug files and directories
3731 * are created. Create the file entries now, and attach them
3732 * to the actual file dentries later.
3734 __trace_early_add_events(tr
);
3736 early_enable_events(tr
, false);
3738 trace_printk_start_comm();
3740 register_event_cmds();
3742 register_trigger_cmds();
3748 * event_trace_enable() is called from trace_event_init() first to
3749 * initialize events and perhaps start any events that are on the
3750 * command line. Unfortunately, there are some events that will not
3751 * start this early, like the system call tracepoints that need
3752 * to set the %SYSCALL_WORK_SYSCALL_TRACEPOINT flag of pid 1. But
3753 * event_trace_enable() is called before pid 1 starts, and this flag
3754 * is never set, making the syscall tracepoint never get reached, but
3755 * the event is enabled regardless (and not doing anything).
3757 static __init
int event_trace_enable_again(void)
3759 struct trace_array
*tr
;
3761 tr
= top_trace_array();
3765 early_enable_events(tr
, true);
3770 early_initcall(event_trace_enable_again
);
3772 /* Init fields which doesn't related to the tracefs */
3773 static __init
int event_trace_init_fields(void)
3775 if (trace_define_generic_fields())
3776 pr_warn("tracing: Failed to allocated generic fields");
3778 if (trace_define_common_fields())
3779 pr_warn("tracing: Failed to allocate common fields");
3784 __init
int event_trace_init(void)
3786 struct trace_array
*tr
;
3789 tr
= top_trace_array();
3793 trace_create_file("available_events", TRACE_MODE_READ
,
3794 NULL
, tr
, &ftrace_avail_fops
);
3796 ret
= early_event_add_tracer(NULL
, tr
);
3800 #ifdef CONFIG_MODULES
3801 ret
= register_module_notifier(&trace_module_nb
);
3803 pr_warn("Failed to register trace events module notifier\n");
3806 eventdir_initialized
= true;
3811 void __init
trace_event_init(void)
3813 event_trace_memsetup();
3814 init_ftrace_syscalls();
3815 event_trace_enable();
3816 event_trace_init_fields();
3819 #ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
3821 static DEFINE_SPINLOCK(test_spinlock
);
3822 static DEFINE_SPINLOCK(test_spinlock_irq
);
3823 static DEFINE_MUTEX(test_mutex
);
3825 static __init
void test_work(struct work_struct
*dummy
)
3827 spin_lock(&test_spinlock
);
3828 spin_lock_irq(&test_spinlock_irq
);
3830 spin_unlock_irq(&test_spinlock_irq
);
3831 spin_unlock(&test_spinlock
);
3833 mutex_lock(&test_mutex
);
3835 mutex_unlock(&test_mutex
);
3838 static __init
int event_test_thread(void *unused
)
3842 test_malloc
= kmalloc(1234, GFP_KERNEL
);
3844 pr_info("failed to kmalloc\n");
3846 schedule_on_each_cpu(test_work
);
3850 set_current_state(TASK_INTERRUPTIBLE
);
3851 while (!kthread_should_stop()) {
3853 set_current_state(TASK_INTERRUPTIBLE
);
3855 __set_current_state(TASK_RUNNING
);
3861 * Do various things that may trigger events.
3863 static __init
void event_test_stuff(void)
3865 struct task_struct
*test_thread
;
3867 test_thread
= kthread_run(event_test_thread
, NULL
, "test-events");
3869 kthread_stop(test_thread
);
3873 * For every trace event defined, we will test each trace point separately,
3874 * and then by groups, and finally all trace points.
3876 static __init
void event_trace_self_tests(void)
3878 struct trace_subsystem_dir
*dir
;
3879 struct trace_event_file
*file
;
3880 struct trace_event_call
*call
;
3881 struct event_subsystem
*system
;
3882 struct trace_array
*tr
;
3885 tr
= top_trace_array();
3889 pr_info("Running tests on trace events:\n");
3891 list_for_each_entry(file
, &tr
->events
, list
) {
3893 call
= file
->event_call
;
3895 /* Only test those that have a probe */
3896 if (!call
->class || !call
->class->probe
)
3900 * Testing syscall events here is pretty useless, but
3901 * we still do it if configured. But this is time consuming.
3902 * What we really need is a user thread to perform the
3903 * syscalls as we test.
3905 #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
3906 if (call
->class->system
&&
3907 strcmp(call
->class->system
, "syscalls") == 0)
3911 pr_info("Testing event %s: ", trace_event_name(call
));
3914 * If an event is already enabled, someone is using
3915 * it and the self test should not be on.
3917 if (file
->flags
& EVENT_FILE_FL_ENABLED
) {
3918 pr_warn("Enabled event during self test!\n");
3923 ftrace_event_enable_disable(file
, 1);
3925 ftrace_event_enable_disable(file
, 0);
3930 /* Now test at the sub system level */
3932 pr_info("Running tests on trace event systems:\n");
3934 list_for_each_entry(dir
, &tr
->systems
, list
) {
3936 system
= dir
->subsystem
;
3938 /* the ftrace system is special, skip it */
3939 if (strcmp(system
->name
, "ftrace") == 0)
3942 pr_info("Testing event system %s: ", system
->name
);
3944 ret
= __ftrace_set_clr_event(tr
, NULL
, system
->name
, NULL
, 1);
3945 if (WARN_ON_ONCE(ret
)) {
3946 pr_warn("error enabling system %s\n",
3953 ret
= __ftrace_set_clr_event(tr
, NULL
, system
->name
, NULL
, 0);
3954 if (WARN_ON_ONCE(ret
)) {
3955 pr_warn("error disabling system %s\n",
3963 /* Test with all events enabled */
3965 pr_info("Running tests on all trace events:\n");
3966 pr_info("Testing all events: ");
3968 ret
= __ftrace_set_clr_event(tr
, NULL
, NULL
, NULL
, 1);
3969 if (WARN_ON_ONCE(ret
)) {
3970 pr_warn("error enabling all events\n");
3977 ret
= __ftrace_set_clr_event(tr
, NULL
, NULL
, NULL
, 0);
3978 if (WARN_ON_ONCE(ret
)) {
3979 pr_warn("error disabling all events\n");
3986 #ifdef CONFIG_FUNCTION_TRACER
3988 static DEFINE_PER_CPU(atomic_t
, ftrace_test_event_disable
);
3990 static struct trace_event_file event_trace_file __initdata
;
3993 function_test_events_call(unsigned long ip
, unsigned long parent_ip
,
3994 struct ftrace_ops
*op
, struct ftrace_regs
*regs
)
3996 struct trace_buffer
*buffer
;
3997 struct ring_buffer_event
*event
;
3998 struct ftrace_entry
*entry
;
3999 unsigned int trace_ctx
;
4003 trace_ctx
= tracing_gen_ctx();
4004 preempt_disable_notrace();
4005 cpu
= raw_smp_processor_id();
4006 disabled
= atomic_inc_return(&per_cpu(ftrace_test_event_disable
, cpu
));
4011 event
= trace_event_buffer_lock_reserve(&buffer
, &event_trace_file
,
4012 TRACE_FN
, sizeof(*entry
),
4016 entry
= ring_buffer_event_data(event
);
4018 entry
->parent_ip
= parent_ip
;
4020 event_trigger_unlock_commit(&event_trace_file
, buffer
, event
,
4023 atomic_dec(&per_cpu(ftrace_test_event_disable
, cpu
));
4024 preempt_enable_notrace();
4027 static struct ftrace_ops trace_ops __initdata
=
4029 .func
= function_test_events_call
,
4032 static __init
void event_trace_self_test_with_function(void)
4036 event_trace_file
.tr
= top_trace_array();
4037 if (WARN_ON(!event_trace_file
.tr
))
4040 ret
= register_ftrace_function(&trace_ops
);
4041 if (WARN_ON(ret
< 0)) {
4042 pr_info("Failed to enable function tracer for event tests\n");
4045 pr_info("Running tests again, along with the function tracer\n");
4046 event_trace_self_tests();
4047 unregister_ftrace_function(&trace_ops
);
4050 static __init
void event_trace_self_test_with_function(void)
4055 static __init
int event_trace_self_tests_init(void)
4057 if (!tracing_selftest_disabled
) {
4058 event_trace_self_tests();
4059 event_trace_self_test_with_function();
4065 late_initcall(event_trace_self_tests_init
);