]> git.ipfire.org Git - thirdparty/kernel/stable.git/blame - kernel/trace/trace_events_hist.c
projects / thirdparty / kernel / stable.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tracing: Prevent hist_field_var_ref() from accessing NULL tracing_map_elts
[thirdparty/kernel/stable.git] / kernel / trace / trace_events_hist.c
CommitLineData
bcea3f96 1// SPDX-License-Identifier: GPL-2.0
7ef224d1
TZ		 2/*
3 * trace_events_hist - trace event hist triggers
4 *
7ef224d1
TZ		 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6 */
7
8#include <linux/module.h>
9#include <linux/kallsyms.h>
10#include <linux/mutex.h>
11#include <linux/slab.h>
12#include <linux/stacktrace.h>
b2d09103 13#include <linux/rculist.h>
4b147936 14#include <linux/tracefs.h>
7ef224d1
TZ		 15
16#include "tracing_map.h"
17#include "trace.h"
18
4b147936
TZ		 19#define SYNTH_SYSTEM "synthetic"
20#define SYNTH_FIELDS_MAX 16
21
22#define STR_VAR_LEN_MAX 32 /* must be multiple of sizeof(u64) */
23
7ef224d1
TZ		 24struct hist_field;
25
df35d93b
TZ		 26typedef u64 (*hist_field_fn_t) (struct hist_field *field,
27 struct tracing_map_elt *elt,
28 struct ring_buffer_event *rbe,
29 void *event);
7ef224d1 30
5819eadd 31#define HIST_FIELD_OPERANDS_MAX 2
30350d65 32#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
0212e2aa 33#define HIST_ACTIONS_MAX 8
30350d65 34
100719dc
TZ		 35enum field_op_id {
36 FIELD_OP_NONE,
37 FIELD_OP_PLUS,
38 FIELD_OP_MINUS,
39 FIELD_OP_UNARY_MINUS,
40};
41
30350d65
TZ		 42struct hist_var {
43 char *name;
44 struct hist_trigger_data *hist_data;
45 unsigned int idx;
46};
5819eadd 47
7ef224d1
TZ		 48struct hist_field {
49 struct ftrace_event_field *field;
50 unsigned long flags;
51 hist_field_fn_t fn;
52 unsigned int size;
76a3b0c8 53 unsigned int offset;
5819eadd 54 unsigned int is_signed;
19a9facd 55 const char *type;
5819eadd 56 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
b559d003 57 struct hist_trigger_data *hist_data;
30350d65 58 struct hist_var var;
100719dc 59 enum field_op_id operator;
067fe038
TZ		 60 char *system;
61 char *event_name;
100719dc 62 char *name;
067fe038
TZ		 63 unsigned int var_idx;
64 unsigned int var_ref_idx;
65 bool read_once;
7ef224d1
TZ		 66};
67
df35d93b
TZ		 68static u64 hist_field_none(struct hist_field *field,
69 struct tracing_map_elt *elt,
70 struct ring_buffer_event *rbe,
71 void *event)
69a0200c
TZ		 72{
73 return 0;
74}
75
df35d93b
TZ		 76static u64 hist_field_counter(struct hist_field *field,
77 struct tracing_map_elt *elt,
78 struct ring_buffer_event *rbe,
79 void *event)
7ef224d1
TZ		 80{
81 return 1;
82}
83
df35d93b
TZ		 84static u64 hist_field_string(struct hist_field *hist_field,
85 struct tracing_map_elt *elt,
86 struct ring_buffer_event *rbe,
87 void *event)
7ef224d1
TZ		 88{
89 char *addr = (char *)(event + hist_field->field->offset);
90
91 return (u64)(unsigned long)addr;
92}
93
df35d93b
TZ		 94static u64 hist_field_dynstring(struct hist_field *hist_field,
95 struct tracing_map_elt *elt,
96 struct ring_buffer_event *rbe,
97 void *event)
79e577cb
NK		 98{
99 u32 str_item = *(u32 *)(event + hist_field->field->offset);
100 int str_loc = str_item & 0xffff;
101 char *addr = (char *)(event + str_loc);
102
103 return (u64)(unsigned long)addr;
104}
105
df35d93b
TZ		 106static u64 hist_field_pstring(struct hist_field *hist_field,
107 struct tracing_map_elt *elt,
108 struct ring_buffer_event *rbe,
109 void *event)
79e577cb
NK		 110{
111 char **addr = (char **)(event + hist_field->field->offset);
112
113 return (u64)(unsigned long)*addr;
114}
115
df35d93b
TZ		 116static u64 hist_field_log2(struct hist_field *hist_field,
117 struct tracing_map_elt *elt,
118 struct ring_buffer_event *rbe,
119 void *event)
4b94f5b7 120{
5819eadd
TZ		 121 struct hist_field *operand = hist_field->operands[0];
122
df35d93b 123 u64 val = operand->fn(operand, elt, rbe, event);
4b94f5b7
NK		 124
125 return (u64) ilog2(roundup_pow_of_two(val));
126}
127
df35d93b
TZ		 128static u64 hist_field_plus(struct hist_field *hist_field,
129 struct tracing_map_elt *elt,
130 struct ring_buffer_event *rbe,
131 void *event)
100719dc
TZ		 132{
133 struct hist_field *operand1 = hist_field->operands[0];
134 struct hist_field *operand2 = hist_field->operands[1];
135
df35d93b
TZ		 136 u64 val1 = operand1->fn(operand1, elt, rbe, event);
137 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 138
139 return val1 + val2;
140}
141
df35d93b
TZ		 142static u64 hist_field_minus(struct hist_field *hist_field,
143 struct tracing_map_elt *elt,
144 struct ring_buffer_event *rbe,
145 void *event)
100719dc
TZ		 146{
147 struct hist_field *operand1 = hist_field->operands[0];
148 struct hist_field *operand2 = hist_field->operands[1];
149
df35d93b
TZ		 150 u64 val1 = operand1->fn(operand1, elt, rbe, event);
151 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 152
153 return val1 - val2;
154}
155
df35d93b
TZ		 156static u64 hist_field_unary_minus(struct hist_field *hist_field,
157 struct tracing_map_elt *elt,
158 struct ring_buffer_event *rbe,
159 void *event)
100719dc
TZ		 160{
161 struct hist_field *operand = hist_field->operands[0];
162
df35d93b 163 s64 sval = (s64)operand->fn(operand, elt, rbe, event);
100719dc
TZ		 164 u64 val = (u64)-sval;
165
166 return val;
167}
168
7ef224d1 169#define DEFINE_HIST_FIELD_FN(type) \
fbd302cb 170 static u64 hist_field_##type(struct hist_field *hist_field, \
df35d93b
TZ		 171 struct tracing_map_elt *elt, \
172 struct ring_buffer_event *rbe, \
173 void *event) \
7ef224d1
TZ		 174{ \
175 type *addr = (type *)(event + hist_field->field->offset); \
176 \
79e577cb 177 return (u64)(unsigned long)*addr; \
7ef224d1
TZ		 178}
179
180DEFINE_HIST_FIELD_FN(s64);
181DEFINE_HIST_FIELD_FN(u64);
182DEFINE_HIST_FIELD_FN(s32);
183DEFINE_HIST_FIELD_FN(u32);
184DEFINE_HIST_FIELD_FN(s16);
185DEFINE_HIST_FIELD_FN(u16);
186DEFINE_HIST_FIELD_FN(s8);
187DEFINE_HIST_FIELD_FN(u8);
188
189#define for_each_hist_field(i, hist_data) \
190 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
191
192#define for_each_hist_val_field(i, hist_data) \
193 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
194
195#define for_each_hist_key_field(i, hist_data) \
196 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
197
69a0200c
TZ		 198#define HIST_STACKTRACE_DEPTH 16
199#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
200#define HIST_STACKTRACE_SKIP 5
201
7ef224d1 202#define HITCOUNT_IDX 0
69a0200c 203#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
7ef224d1
TZ		 204
205enum hist_field_flags {
0d7a8325
TZ		 206 HIST_FIELD_FL_HITCOUNT = 1 << 0,
207 HIST_FIELD_FL_KEY = 1 << 1,
208 HIST_FIELD_FL_STRING = 1 << 2,
209 HIST_FIELD_FL_HEX = 1 << 3,
210 HIST_FIELD_FL_SYM = 1 << 4,
211 HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
212 HIST_FIELD_FL_EXECNAME = 1 << 6,
213 HIST_FIELD_FL_SYSCALL = 1 << 7,
214 HIST_FIELD_FL_STACKTRACE = 1 << 8,
215 HIST_FIELD_FL_LOG2 = 1 << 9,
ad42febe 216 HIST_FIELD_FL_TIMESTAMP = 1 << 10,
860f9f6b 217 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
30350d65 218 HIST_FIELD_FL_VAR = 1 << 12,
100719dc 219 HIST_FIELD_FL_EXPR = 1 << 13,
067fe038 220 HIST_FIELD_FL_VAR_REF = 1 << 14,
8b7622bf 221 HIST_FIELD_FL_CPU = 1 << 15,
7e8b88a3 222 HIST_FIELD_FL_ALIAS = 1 << 16,
30350d65
TZ		 223};
224
225struct var_defs {
226 unsigned int n_vars;
227 char *name[TRACING_MAP_VARS_MAX];
228 char *expr[TRACING_MAP_VARS_MAX];
7ef224d1
TZ		 229};
230
231struct hist_trigger_attrs {
232 char *keys_str;
f2606835 233 char *vals_str;
e62347d2 234 char *sort_key_str;
5463bfda 235 char *name;
a4072fe8 236 char *clock;
83e99914
TZ		 237 bool pause;
238 bool cont;
e86ae9ba 239 bool clear;
860f9f6b 240 bool ts_in_usecs;
7ef224d1 241 unsigned int map_bits;
30350d65
TZ		 242
243 char *assignment_str[TRACING_MAP_VARS_MAX];
244 unsigned int n_assignments;
245
0212e2aa
TZ		 246 char *action_str[HIST_ACTIONS_MAX];
247 unsigned int n_actions;
248
30350d65 249 struct var_defs var_defs;
7ef224d1
TZ		 250};
251
02205a67
TZ		 252struct field_var {
253 struct hist_field *var;
254 struct hist_field *val;
255};
256
257struct field_var_hist {
258 struct hist_trigger_data *hist_data;
259 char *cmd;
260};
261
7ef224d1 262struct hist_trigger_data {
30350d65 263 struct hist_field *fields[HIST_FIELDS_MAX];
7ef224d1
TZ		 264 unsigned int n_vals;
265 unsigned int n_keys;
266 unsigned int n_fields;
30350d65 267 unsigned int n_vars;
7ef224d1
TZ		 268 unsigned int key_size;
269 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
270 unsigned int n_sort_keys;
271 struct trace_event_file *event_file;
272 struct hist_trigger_attrs *attrs;
273 struct tracing_map *map;
ad42febe 274 bool enable_timestamps;
30350d65 275 bool remove;
067fe038
TZ		 276 struct hist_field *var_refs[TRACING_MAP_VARS_MAX];
277 unsigned int n_var_refs;
0212e2aa
TZ		 278
279 struct action_data *actions[HIST_ACTIONS_MAX];
280 unsigned int n_actions;
02205a67 281
c282a386
TZ		 282 struct hist_field *synth_var_refs[SYNTH_FIELDS_MAX];
283 unsigned int n_synth_var_refs;
02205a67
TZ		 284 struct field_var *field_vars[SYNTH_FIELDS_MAX];
285 unsigned int n_field_vars;
286 unsigned int n_field_var_str;
287 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX];
288 unsigned int n_field_var_hists;
50450603
TZ		 289
290 struct field_var *max_vars[SYNTH_FIELDS_MAX];
291 unsigned int n_max_vars;
292 unsigned int n_max_var_str;
0212e2aa
TZ		 293};
294
4b147936
TZ		 295struct synth_field {
296 char *type;
297 char *name;
298 size_t size;
299 bool is_signed;
300 bool is_string;
301};
302
303struct synth_event {
304 struct list_head list;
305 int ref;
306 char *name;
307 struct synth_field **fields;
308 unsigned int n_fields;
309 unsigned int n_u64;
310 struct trace_event_class class;
311 struct trace_event_call call;
312 struct tracepoint *tp;
313};
314
0212e2aa
TZ		 315struct action_data;
316
317typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
318 struct tracing_map_elt *elt, void *rec,
319 struct ring_buffer_event *rbe,
320 struct action_data *data, u64 *var_ref_vals);
321
322struct action_data {
323 action_fn_t fn;
c282a386
TZ		 324 unsigned int n_params;
325 char *params[SYNTH_FIELDS_MAX];
326
327 union {
328 struct {
329 unsigned int var_ref_idx;
330 char *match_event;
331 char *match_event_system;
332 char *synth_event_name;
333 struct synth_event *synth_event;
334 } onmatch;
50450603
TZ		 335
336 struct {
337 char *var_str;
338 char *fn_name;
339 unsigned int max_var_ref_idx;
340 struct hist_field *max_var;
341 struct hist_field *var;
342 } onmax;
c282a386 343 };
7ef224d1
TZ		 344};
345
f404da6e
TZ		 346
347static char last_hist_cmd[MAX_FILTER_STR_VAL];
348static char hist_err_str[MAX_FILTER_STR_VAL];
349
350static void last_cmd_set(char *str)
351{
352 if (!str)
353 return;
354
355 strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
356}
357
358static void hist_err(char *str, char *var)
359{
360 int maxlen = MAX_FILTER_STR_VAL - 1;
361
362 if (!str)
363 return;
364
365 if (strlen(hist_err_str))
366 return;
367
368 if (!var)
369 var = "";
370
371 if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
372 return;
373
374 strcat(hist_err_str, str);
375 strcat(hist_err_str, var);
376}
377
378static void hist_err_event(char *str, char *system, char *event, char *var)
379{
380 char err[MAX_FILTER_STR_VAL];
381
382 if (system && var)
383 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
384 else if (system)
385 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
386 else
cf4d418e 387 strscpy(err, var, MAX_FILTER_STR_VAL);
f404da6e
TZ		 388
389 hist_err(str, err);
390}
391
392static void hist_err_clear(void)
393{
394 hist_err_str[0] = '\0';
395}
396
397static bool have_hist_err(void)
398{
399 if (strlen(hist_err_str))
400 return true;
401
402 return false;
403}
404
4b147936
TZ		 405static LIST_HEAD(synth_event_list);
406static DEFINE_MUTEX(synth_event_mutex);
407
408struct synth_trace_event {
409 struct trace_entry ent;
410 u64 fields[];
411};
412
413static int synth_event_define_fields(struct trace_event_call *call)
414{
415 struct synth_trace_event trace;
416 int offset = offsetof(typeof(trace), fields);
417 struct synth_event *event = call->data;
418 unsigned int i, size, n_u64;
419 char *name, *type;
420 bool is_signed;
421 int ret = 0;
422
423 for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
424 size = event->fields[i]->size;
425 is_signed = event->fields[i]->is_signed;
426 type = event->fields[i]->type;
427 name = event->fields[i]->name;
428 ret = trace_define_field(call, type, name, offset, size,
429 is_signed, FILTER_OTHER);
430 if (ret)
431 break;
432
433 if (event->fields[i]->is_string) {
434 offset += STR_VAR_LEN_MAX;
435 n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
436 } else {
437 offset += sizeof(u64);
438 n_u64++;
439 }
440 }
441
442 event->n_u64 = n_u64;
443
444 return ret;
445}
446
447static bool synth_field_signed(char *type)
448{
449 if (strncmp(type, "u", 1) == 0)
450 return false;
451
452 return true;
453}
454
455static int synth_field_is_string(char *type)
456{
457 if (strstr(type, "char[") != NULL)
458 return true;
459
460 return false;
461}
462
463static int synth_field_string_size(char *type)
464{
465 char buf[4], *end, *start;
466 unsigned int len;
467 int size, err;
468
469 start = strstr(type, "char[");
470 if (start == NULL)
471 return -EINVAL;
472 start += strlen("char[");
473
474 end = strchr(type, ']');
475 if (!end || end < start)
476 return -EINVAL;
477
478 len = end - start;
479 if (len > 3)
480 return -EINVAL;
481
482 strncpy(buf, start, len);
483 buf[len] = '\0';
484
485 err = kstrtouint(buf, 0, &size);
486 if (err)
487 return err;
488
489 if (size > STR_VAR_LEN_MAX)
490 return -EINVAL;
491
492 return size;
493}
494
495static int synth_field_size(char *type)
496{
497 int size = 0;
498
499 if (strcmp(type, "s64") == 0)
500 size = sizeof(s64);
501 else if (strcmp(type, "u64") == 0)
502 size = sizeof(u64);
503 else if (strcmp(type, "s32") == 0)
504 size = sizeof(s32);
505 else if (strcmp(type, "u32") == 0)
506 size = sizeof(u32);
507 else if (strcmp(type, "s16") == 0)
508 size = sizeof(s16);
509 else if (strcmp(type, "u16") == 0)
510 size = sizeof(u16);
511 else if (strcmp(type, "s8") == 0)
512 size = sizeof(s8);
513 else if (strcmp(type, "u8") == 0)
514 size = sizeof(u8);
515 else if (strcmp(type, "char") == 0)
516 size = sizeof(char);
517 else if (strcmp(type, "unsigned char") == 0)
518 size = sizeof(unsigned char);
519 else if (strcmp(type, "int") == 0)
520 size = sizeof(int);
521 else if (strcmp(type, "unsigned int") == 0)
522 size = sizeof(unsigned int);
523 else if (strcmp(type, "long") == 0)
524 size = sizeof(long);
525 else if (strcmp(type, "unsigned long") == 0)
526 size = sizeof(unsigned long);
527 else if (strcmp(type, "pid_t") == 0)
528 size = sizeof(pid_t);
529 else if (synth_field_is_string(type))
530 size = synth_field_string_size(type);
531
532 return size;
533}
534
535static const char *synth_field_fmt(char *type)
536{
537 const char *fmt = "%llu";
538
539 if (strcmp(type, "s64") == 0)
540 fmt = "%lld";
541 else if (strcmp(type, "u64") == 0)
542 fmt = "%llu";
543 else if (strcmp(type, "s32") == 0)
544 fmt = "%d";
545 else if (strcmp(type, "u32") == 0)
546 fmt = "%u";
547 else if (strcmp(type, "s16") == 0)
548 fmt = "%d";
549 else if (strcmp(type, "u16") == 0)
550 fmt = "%u";
551 else if (strcmp(type, "s8") == 0)
552 fmt = "%d";
553 else if (strcmp(type, "u8") == 0)
554 fmt = "%u";
555 else if (strcmp(type, "char") == 0)
556 fmt = "%d";
557 else if (strcmp(type, "unsigned char") == 0)
558 fmt = "%u";
559 else if (strcmp(type, "int") == 0)
560 fmt = "%d";
561 else if (strcmp(type, "unsigned int") == 0)
562 fmt = "%u";
563 else if (strcmp(type, "long") == 0)
564 fmt = "%ld";
565 else if (strcmp(type, "unsigned long") == 0)
566 fmt = "%lu";
567 else if (strcmp(type, "pid_t") == 0)
568 fmt = "%d";
569 else if (synth_field_is_string(type))
570 fmt = "%s";
571
572 return fmt;
573}
574
575static enum print_line_t print_synth_event(struct trace_iterator *iter,
576 int flags,
577 struct trace_event *event)
578{
579 struct trace_array *tr = iter->tr;
580 struct trace_seq *s = &iter->seq;
581 struct synth_trace_event *entry;
582 struct synth_event *se;
583 unsigned int i, n_u64;
584 char print_fmt[32];
585 const char *fmt;
586
587 entry = (struct synth_trace_event *)iter->ent;
588 se = container_of(event, struct synth_event, call.event);
589
590 trace_seq_printf(s, "%s: ", se->name);
591
592 for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
593 if (trace_seq_has_overflowed(s))
594 goto end;
595
596 fmt = synth_field_fmt(se->fields[i]->type);
597
598 /* parameter types */
599 if (tr->trace_flags & TRACE_ITER_VERBOSE)
600 trace_seq_printf(s, "%s ", fmt);
601
602 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
603
604 /* parameter values */
605 if (se->fields[i]->is_string) {
606 trace_seq_printf(s, print_fmt, se->fields[i]->name,
607 (char *)&entry->fields[n_u64],
608 i == se->n_fields - 1 ? "" : " ");
609 n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
610 } else {
611 trace_seq_printf(s, print_fmt, se->fields[i]->name,
612 entry->fields[n_u64],
613 i == se->n_fields - 1 ? "" : " ");
614 n_u64++;
615 }
616 }
617end:
618 trace_seq_putc(s, '\n');
619
620 return trace_handle_return(s);
621}
622
623static struct trace_event_functions synth_event_funcs = {
624 .trace = print_synth_event
625};
626
627static notrace void trace_event_raw_event_synth(void *__data,
628 u64 *var_ref_vals,
629 unsigned int var_ref_idx)
630{
631 struct trace_event_file *trace_file = __data;
632 struct synth_trace_event *entry;
633 struct trace_event_buffer fbuffer;
4708abc6 634 struct ring_buffer *buffer;
4b147936
TZ		 635 struct synth_event *event;
636 unsigned int i, n_u64;
637 int fields_size = 0;
638
639 event = trace_file->event_call->data;
640
641 if (trace_trigger_soft_disabled(trace_file))
642 return;
643
644 fields_size = event->n_u64 * sizeof(u64);
645
4708abc6
SRV		 646 /*
647 * Avoid ring buffer recursion detection, as this event
648 * is being performed within another event.
649 */
650 buffer = trace_file->tr->trace_buffer.buffer;
651 ring_buffer_nest_start(buffer);
652
4b147936
TZ		 653 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
654 sizeof(*entry) + fields_size);
655 if (!entry)
4708abc6 656 goto out;
4b147936
TZ		 657
658 for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
659 if (event->fields[i]->is_string) {
660 char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
661 char *str_field = (char *)&entry->fields[n_u64];
662
ad452870 663 strscpy(str_field, str_val, STR_VAR_LEN_MAX);
4b147936
TZ		 664 n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
665 } else {
666 entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
667 n_u64++;
668 }
669 }
670
671 trace_event_buffer_commit(&fbuffer);
4708abc6
SRV		 672out:
673 ring_buffer_nest_end(buffer);
4b147936
TZ		 674}
675
676static void free_synth_event_print_fmt(struct trace_event_call *call)
677{
678 if (call) {
679 kfree(call->print_fmt);
680 call->print_fmt = NULL;
681 }
682}
683
684static int __set_synth_event_print_fmt(struct synth_event *event,
685 char *buf, int len)
686{
687 const char *fmt;
688 int pos = 0;
689 int i;
690
691 /* When len=0, we just calculate the needed length */
692#define LEN_OR_ZERO (len ? len - pos : 0)
693
694 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
695 for (i = 0; i < event->n_fields; i++) {
696 fmt = synth_field_fmt(event->fields[i]->type);
697 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
698 event->fields[i]->name, fmt,
699 i == event->n_fields - 1 ? "" : ", ");
700 }
701 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
702
703 for (i = 0; i < event->n_fields; i++) {
704 pos += snprintf(buf + pos, LEN_OR_ZERO,
705 ", REC->%s", event->fields[i]->name);
706 }
707
708#undef LEN_OR_ZERO
709
710 /* return the length of print_fmt */
711 return pos;
712}
713
714static int set_synth_event_print_fmt(struct trace_event_call *call)
715{
716 struct synth_event *event = call->data;
717 char *print_fmt;
718 int len;
719
720 /* First: called with 0 length to calculate the needed length */
721 len = __set_synth_event_print_fmt(event, NULL, 0);
722
723 print_fmt = kmalloc(len + 1, GFP_KERNEL);
724 if (!print_fmt)
725 return -ENOMEM;
726
727 /* Second: actually write the @print_fmt */
728 __set_synth_event_print_fmt(event, print_fmt, len + 1);
729 call->print_fmt = print_fmt;
730
731 return 0;
732}
733
734static void free_synth_field(struct synth_field *field)
735{
736 kfree(field->type);
737 kfree(field->name);
738 kfree(field);
739}
740
282447ba
MH		 741static struct synth_field *parse_synth_field(int argc, char **argv,
742 int *consumed)
4b147936
TZ		 743{
744 struct synth_field *field;
282447ba
MH		 745 const char *prefix = NULL;
746 char *field_type = argv[0], *field_name;
4b147936
TZ		 747 int len, ret = 0;
748 char *array;
749
750 if (field_type[0] == ';')
751 field_type++;
752
282447ba
MH		 753 if (!strcmp(field_type, "unsigned")) {
754 if (argc < 3)
755 return ERR_PTR(-EINVAL);
756 prefix = "unsigned ";
757 field_type = argv[1];
758 field_name = argv[2];
759 *consumed = 3;
760 } else {
761 field_name = argv[1];
762 *consumed = 2;
763 }
764
4b147936
TZ		 765 len = strlen(field_name);
766 if (field_name[len - 1] == ';')
767 field_name[len - 1] = '\0';
768
769 field = kzalloc(sizeof(*field), GFP_KERNEL);
770 if (!field)
771 return ERR_PTR(-ENOMEM);
772
773 len = strlen(field_type) + 1;
774 array = strchr(field_name, '[');
775 if (array)
776 len += strlen(array);
282447ba
MH		 777 if (prefix)
778 len += strlen(prefix);
4b147936
TZ		 779 field->type = kzalloc(len, GFP_KERNEL);
780 if (!field->type) {
781 ret = -ENOMEM;
782 goto free;
783 }
282447ba
MH		 784 if (prefix)
785 strcat(field->type, prefix);
4b147936
TZ		 786 strcat(field->type, field_type);
787 if (array) {
788 strcat(field->type, array);
789 *array = '\0';
790 }
791
792 field->size = synth_field_size(field->type);
793 if (!field->size) {
794 ret = -EINVAL;
795 goto free;
796 }
797
798 if (synth_field_is_string(field->type))
799 field->is_string = true;
800
801 field->is_signed = synth_field_signed(field->type);
802
803 field->name = kstrdup(field_name, GFP_KERNEL);
804 if (!field->name) {
805 ret = -ENOMEM;
806 goto free;
807 }
808 out:
809 return field;
810 free:
811 free_synth_field(field);
812 field = ERR_PTR(ret);
813 goto out;
814}
815
816static void free_synth_tracepoint(struct tracepoint *tp)
817{
818 if (!tp)
819 return;
820
821 kfree(tp->name);
822 kfree(tp);
823}
824
825static struct tracepoint *alloc_synth_tracepoint(char *name)
826{
827 struct tracepoint *tp;
828
829 tp = kzalloc(sizeof(*tp), GFP_KERNEL);
830 if (!tp)
831 return ERR_PTR(-ENOMEM);
832
833 tp->name = kstrdup(name, GFP_KERNEL);
834 if (!tp->name) {
835 kfree(tp);
836 return ERR_PTR(-ENOMEM);
837 }
838
839 return tp;
840}
841
842typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
843 unsigned int var_ref_idx);
844
845static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
846 unsigned int var_ref_idx)
847{
848 struct tracepoint *tp = event->tp;
849
850 if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
851 struct tracepoint_func *probe_func_ptr;
852 synth_probe_func_t probe_func;
853 void *__data;
854
855 if (!(cpu_online(raw_smp_processor_id())))
856 return;
857
858 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
859 if (probe_func_ptr) {
860 do {
861 probe_func = probe_func_ptr->func;
862 __data = probe_func_ptr->data;
863 probe_func(__data, var_ref_vals, var_ref_idx);
864 } while ((++probe_func_ptr)->func);
865 }
866 }
867}
868
869static struct synth_event *find_synth_event(const char *name)
870{
871 struct synth_event *event;
872
873 list_for_each_entry(event, &synth_event_list, list) {
874 if (strcmp(event->name, name) == 0)
875 return event;
876 }
877
878 return NULL;
879}
880
881static int register_synth_event(struct synth_event *event)
882{
883 struct trace_event_call *call = &event->call;
884 int ret = 0;
885
886 event->call.class = &event->class;
887 event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
888 if (!event->class.system) {
889 ret = -ENOMEM;
890 goto out;
891 }
892
893 event->tp = alloc_synth_tracepoint(event->name);
894 if (IS_ERR(event->tp)) {
895 ret = PTR_ERR(event->tp);
896 event->tp = NULL;
897 goto out;
898 }
899
900 INIT_LIST_HEAD(&call->class->fields);
901 call->event.funcs = &synth_event_funcs;
902 call->class->define_fields = synth_event_define_fields;
903
904 ret = register_trace_event(&call->event);
905 if (!ret) {
906 ret = -ENODEV;
907 goto out;
908 }
909 call->flags = TRACE_EVENT_FL_TRACEPOINT;
910 call->class->reg = trace_event_reg;
911 call->class->probe = trace_event_raw_event_synth;
912 call->data = event;
913 call->tp = event->tp;
914
915 ret = trace_add_event_call(call);
916 if (ret) {
917 pr_warn("Failed to register synthetic event: %s\n",
918 trace_event_name(call));
919 goto err;
920 }
921
922 ret = set_synth_event_print_fmt(call);
923 if (ret < 0) {
924 trace_remove_event_call(call);
925 goto err;
926 }
927 out:
928 return ret;
929 err:
930 unregister_trace_event(&call->event);
931 goto out;
932}
933
934static int unregister_synth_event(struct synth_event *event)
935{
936 struct trace_event_call *call = &event->call;
937 int ret;
938
939 ret = trace_remove_event_call(call);
940
941 return ret;
942}
943
944static void free_synth_event(struct synth_event *event)
945{
946 unsigned int i;
947
948 if (!event)
949 return;
950
951 for (i = 0; i < event->n_fields; i++)
952 free_synth_field(event->fields[i]);
953
954 kfree(event->fields);
955 kfree(event->name);
956 kfree(event->class.system);
957 free_synth_tracepoint(event->tp);
958 free_synth_event_print_fmt(&event->call);
959 kfree(event);
960}
961
962static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
963 struct synth_field **fields)
964{
965 struct synth_event *event;
966 unsigned int i;
967
968 event = kzalloc(sizeof(*event), GFP_KERNEL);
969 if (!event) {
970 event = ERR_PTR(-ENOMEM);
971 goto out;
972 }
973
974 event->name = kstrdup(event_name, GFP_KERNEL);
975 if (!event->name) {
976 kfree(event);
977 event = ERR_PTR(-ENOMEM);
978 goto out;
979 }
980
981 event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
982 if (!event->fields) {
983 free_synth_event(event);
984 event = ERR_PTR(-ENOMEM);
985 goto out;
986 }
987
988 for (i = 0; i < n_fields; i++)
989 event->fields[i] = fields[i];
990
991 event->n_fields = n_fields;
992 out:
993 return event;
994}
995
c282a386
TZ		 996static void action_trace(struct hist_trigger_data *hist_data,
997 struct tracing_map_elt *elt, void *rec,
998 struct ring_buffer_event *rbe,
999 struct action_data *data, u64 *var_ref_vals)
1000{
1001 struct synth_event *event = data->onmatch.synth_event;
1002
1003 trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
1004}
1005
1006struct hist_var_data {
1007 struct list_head list;
1008 struct hist_trigger_data *hist_data;
1009};
1010
4b147936
TZ		 1011static void add_or_delete_synth_event(struct synth_event *event, int delete)
1012{
1013 if (delete)
1014 free_synth_event(event);
1015 else {
1016 mutex_lock(&synth_event_mutex);
1017 if (!find_synth_event(event->name))
1018 list_add(&event->list, &synth_event_list);
1019 else
1020 free_synth_event(event);
1021 mutex_unlock(&synth_event_mutex);
1022 }
1023}
1024
1025static int create_synth_event(int argc, char **argv)
1026{
1027 struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1028 struct synth_event *event = NULL;
1029 bool delete_event = false;
282447ba 1030 int i, consumed = 0, n_fields = 0, ret = 0;
4b147936
TZ		 1031 char *name;
1032
1033 mutex_lock(&synth_event_mutex);
1034
1035 /*
1036 * Argument syntax:
1037 * - Add synthetic event: <event_name> field[;field] ...
1038 * - Remove synthetic event: !<event_name> field[;field] ...
1039 * where 'field' = type field_name
1040 */
1041 if (argc < 1) {
1042 ret = -EINVAL;
1043 goto out;
1044 }
1045
1046 name = argv[0];
1047 if (name[0] == '!') {
1048 delete_event = true;
1049 name++;
1050 }
1051
1052 event = find_synth_event(name);
1053 if (event) {
1054 if (delete_event) {
1055 if (event->ref) {
1056 event = NULL;
1057 ret = -EBUSY;
1058 goto out;
1059 }
1060 list_del(&event->list);
1061 goto out;
1062 }
1063 event = NULL;
1064 ret = -EEXIST;
1065 goto out;
2b3171ec
MH		 1066 } else if (delete_event) {
1067 ret = -ENOENT;
4b147936 1068 goto out;
2b3171ec 1069 }
4b147936
TZ		 1070
1071 if (argc < 2) {
1072 ret = -EINVAL;
1073 goto out;
1074 }
1075
1076 for (i = 1; i < argc - 1; i++) {
1077 if (strcmp(argv[i], ";") == 0)
1078 continue;
1079 if (n_fields == SYNTH_FIELDS_MAX) {
1080 ret = -EINVAL;
1081 goto err;
1082 }
1083
282447ba 1084 field = parse_synth_field(argc - i, &argv[i], &consumed);
4b147936
TZ		 1085 if (IS_ERR(field)) {
1086 ret = PTR_ERR(field);
1087 goto err;
1088 }
282447ba
MH		 1089 fields[n_fields++] = field;
1090 i += consumed - 1;
4b147936
TZ		 1091 }
1092
a360d9e4 1093 if (i < argc && strcmp(argv[i], ";") != 0) {
4b147936
TZ		 1094 ret = -EINVAL;
1095 goto err;
1096 }
1097
1098 event = alloc_synth_event(name, n_fields, fields);
1099 if (IS_ERR(event)) {
1100 ret = PTR_ERR(event);
1101 event = NULL;
1102 goto err;
1103 }
1104 out:
1105 mutex_unlock(&synth_event_mutex);
1106
1107 if (event) {
1108 if (delete_event) {
1109 ret = unregister_synth_event(event);
1110 add_or_delete_synth_event(event, !ret);
1111 } else {
1112 ret = register_synth_event(event);
1113 add_or_delete_synth_event(event, ret);
1114 }
1115 }
1116
1117 return ret;
1118 err:
1119 mutex_unlock(&synth_event_mutex);
1120
1121 for (i = 0; i < n_fields; i++)
1122 free_synth_field(fields[i]);
1123 free_synth_event(event);
1124
1125 return ret;
1126}
1127
1128static int release_all_synth_events(void)
1129{
1130 struct list_head release_events;
1131 struct synth_event *event, *e;
1132 int ret = 0;
1133
1134 INIT_LIST_HEAD(&release_events);
1135
1136 mutex_lock(&synth_event_mutex);
1137
1138 list_for_each_entry(event, &synth_event_list, list) {
1139 if (event->ref) {
1140 mutex_unlock(&synth_event_mutex);
1141 return -EBUSY;
1142 }
1143 }
1144
1145 list_splice_init(&event->list, &release_events);
1146
1147 mutex_unlock(&synth_event_mutex);
1148
1149 list_for_each_entry_safe(event, e, &release_events, list) {
1150 list_del(&event->list);
1151
1152 ret = unregister_synth_event(event);
1153 add_or_delete_synth_event(event, !ret);
1154 }
1155
1156 return ret;
1157}
1158
1159
1160static void *synth_events_seq_start(struct seq_file *m, loff_t *pos)
1161{
1162 mutex_lock(&synth_event_mutex);
1163
1164 return seq_list_start(&synth_event_list, *pos);
1165}
1166
1167static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos)
1168{
1169 return seq_list_next(v, &synth_event_list, pos);
1170}
1171
1172static void synth_events_seq_stop(struct seq_file *m, void *v)
1173{
1174 mutex_unlock(&synth_event_mutex);
1175}
1176
1177static int synth_events_seq_show(struct seq_file *m, void *v)
1178{
1179 struct synth_field *field;
1180 struct synth_event *event = v;
1181 unsigned int i;
1182
1183 seq_printf(m, "%s\t", event->name);
1184
1185 for (i = 0; i < event->n_fields; i++) {
1186 field = event->fields[i];
1187
1188 /* parameter values */
1189 seq_printf(m, "%s %s%s", field->type, field->name,
1190 i == event->n_fields - 1 ? "" : "; ");
1191 }
1192
1193 seq_putc(m, '\n');
1194
1195 return 0;
1196}
1197
1198static const struct seq_operations synth_events_seq_op = {
1199 .start = synth_events_seq_start,
1200 .next = synth_events_seq_next,
1201 .stop = synth_events_seq_stop,
1202 .show = synth_events_seq_show
1203};
1204
1205static int synth_events_open(struct inode *inode, struct file *file)
1206{
1207 int ret;
1208
1209 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1210 ret = release_all_synth_events();
1211 if (ret < 0)
1212 return ret;
1213 }
1214
1215 return seq_open(file, &synth_events_seq_op);
1216}
1217
1218static ssize_t synth_events_write(struct file *file,
1219 const char __user *buffer,
1220 size_t count, loff_t *ppos)
1221{
1222 return trace_parse_run_command(file, buffer, count, ppos,
1223 create_synth_event);
1224}
1225
1226static const struct file_operations synth_events_fops = {
1227 .open = synth_events_open,
1228 .write = synth_events_write,
1229 .read = seq_read,
1230 .llseek = seq_lseek,
1231 .release = seq_release,
1232};
1233
df35d93b
TZ		 1234static u64 hist_field_timestamp(struct hist_field *hist_field,
1235 struct tracing_map_elt *elt,
1236 struct ring_buffer_event *rbe,
1237 void *event)
860f9f6b
TZ		 1238{
1239 struct hist_trigger_data *hist_data = hist_field->hist_data;
1240 struct trace_array *tr = hist_data->event_file->tr;
1241
1242 u64 ts = ring_buffer_event_time_stamp(rbe);
1243
1244 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1245 ts = ns2usecs(ts);
1246
1247 return ts;
1248}
1249
8b7622bf
TZ		 1250static u64 hist_field_cpu(struct hist_field *hist_field,
1251 struct tracing_map_elt *elt,
1252 struct ring_buffer_event *rbe,
1253 void *event)
1254{
1255 int cpu = smp_processor_id();
1256
1257 return cpu;
1258}
1259
067fe038
TZ		 1260static struct hist_field *
1261check_field_for_var_ref(struct hist_field *hist_field,
1262 struct hist_trigger_data *var_data,
1263 unsigned int var_idx)
1264{
1265 struct hist_field *found = NULL;
1266
1267 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1268 if (hist_field->var.idx == var_idx &&
1269 hist_field->var.hist_data == var_data) {
1270 found = hist_field;
1271 }
1272 }
1273
1274 return found;
1275}
1276
1277static struct hist_field *
1278check_field_for_var_refs(struct hist_trigger_data *hist_data,
1279 struct hist_field *hist_field,
1280 struct hist_trigger_data *var_data,
1281 unsigned int var_idx,
1282 unsigned int level)
1283{
1284 struct hist_field *found = NULL;
1285 unsigned int i;
1286
1287 if (level > 3)
1288 return found;
1289
1290 if (!hist_field)
1291 return found;
1292
1293 found = check_field_for_var_ref(hist_field, var_data, var_idx);
1294 if (found)
1295 return found;
1296
1297 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1298 struct hist_field *operand;
1299
1300 operand = hist_field->operands[i];
1301 found = check_field_for_var_refs(hist_data, operand, var_data,
1302 var_idx, level + 1);
1303 if (found)
1304 return found;
1305 }
1306
1307 return found;
1308}
1309
1310static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1311 struct hist_trigger_data *var_data,
1312 unsigned int var_idx)
1313{
1314 struct hist_field *hist_field, *found = NULL;
1315 unsigned int i;
1316
1317 for_each_hist_field(i, hist_data) {
1318 hist_field = hist_data->fields[i];
1319 found = check_field_for_var_refs(hist_data, hist_field,
1320 var_data, var_idx, 0);
1321 if (found)
1322 return found;
1323 }
1324
c282a386
TZ		 1325 for (i = 0; i < hist_data->n_synth_var_refs; i++) {
1326 hist_field = hist_data->synth_var_refs[i];
1327 found = check_field_for_var_refs(hist_data, hist_field,
1328 var_data, var_idx, 0);
1329 if (found)
1330 return found;
1331 }
1332
067fe038
TZ		 1333 return found;
1334}
1335
1336static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1337 unsigned int var_idx)
1338{
1339 struct trace_array *tr = hist_data->event_file->tr;
1340 struct hist_field *found = NULL;
1341 struct hist_var_data *var_data;
1342
1343 list_for_each_entry(var_data, &tr->hist_vars, list) {
1344 if (var_data->hist_data == hist_data)
1345 continue;
1346 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1347 if (found)
1348 break;
1349 }
1350
1351 return found;
1352}
1353
1354static bool check_var_refs(struct hist_trigger_data *hist_data)
1355{
1356 struct hist_field *field;
1357 bool found = false;
1358 int i;
1359
1360 for_each_hist_field(i, hist_data) {
1361 field = hist_data->fields[i];
1362 if (field && field->flags & HIST_FIELD_FL_VAR) {
1363 if (find_any_var_ref(hist_data, field->var.idx)) {
1364 found = true;
1365 break;
1366 }
1367 }
1368 }
1369
1370 return found;
1371}
1372
1373static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1374{
1375 struct trace_array *tr = hist_data->event_file->tr;
1376 struct hist_var_data *var_data, *found = NULL;
1377
1378 list_for_each_entry(var_data, &tr->hist_vars, list) {
1379 if (var_data->hist_data == hist_data) {
1380 found = var_data;
1381 break;
1382 }
1383 }
1384
1385 return found;
1386}
1387
1388static bool field_has_hist_vars(struct hist_field *hist_field,
1389 unsigned int level)
1390{
1391 int i;
1392
1393 if (level > 3)
1394 return false;
1395
1396 if (!hist_field)
1397 return false;
1398
1399 if (hist_field->flags & HIST_FIELD_FL_VAR ||
1400 hist_field->flags & HIST_FIELD_FL_VAR_REF)
1401 return true;
1402
1403 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1404 struct hist_field *operand;
1405
1406 operand = hist_field->operands[i];
1407 if (field_has_hist_vars(operand, level + 1))
1408 return true;
1409 }
1410
1411 return false;
1412}
1413
1414static bool has_hist_vars(struct hist_trigger_data *hist_data)
1415{
1416 struct hist_field *hist_field;
1417 int i;
1418
1419 for_each_hist_field(i, hist_data) {
1420 hist_field = hist_data->fields[i];
1421 if (field_has_hist_vars(hist_field, 0))
1422 return true;
1423 }
1424
1425 return false;
1426}
1427
1428static int save_hist_vars(struct hist_trigger_data *hist_data)
1429{
1430 struct trace_array *tr = hist_data->event_file->tr;
1431 struct hist_var_data *var_data;
1432
1433 var_data = find_hist_vars(hist_data);
1434 if (var_data)
1435 return 0;
1436
1437 if (trace_array_get(tr) < 0)
1438 return -ENODEV;
1439
1440 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1441 if (!var_data) {
1442 trace_array_put(tr);
1443 return -ENOMEM;
1444 }
1445
1446 var_data->hist_data = hist_data;
1447 list_add(&var_data->list, &tr->hist_vars);
1448
1449 return 0;
1450}
1451
1452static void remove_hist_vars(struct hist_trigger_data *hist_data)
1453{
1454 struct trace_array *tr = hist_data->event_file->tr;
1455 struct hist_var_data *var_data;
1456
1457 var_data = find_hist_vars(hist_data);
1458 if (!var_data)
1459 return;
1460
1461 if (WARN_ON(check_var_refs(hist_data)))
1462 return;
1463
1464 list_del(&var_data->list);
1465
1466 kfree(var_data);
1467
1468 trace_array_put(tr);
1469}
1470
30350d65
TZ		 1471static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1472 const char *var_name)
1473{
1474 struct hist_field *hist_field, *found = NULL;
1475 int i;
1476
1477 for_each_hist_field(i, hist_data) {
1478 hist_field = hist_data->fields[i];
1479 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1480 strcmp(hist_field->var.name, var_name) == 0) {
1481 found = hist_field;
1482 break;
1483 }
1484 }
1485
1486 return found;
1487}
1488
1489static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1490 struct trace_event_file *file,
1491 const char *var_name)
1492{
1493 struct hist_trigger_data *test_data;
1494 struct event_trigger_data *test;
1495 struct hist_field *hist_field;
1496
1497 hist_field = find_var_field(hist_data, var_name);
1498 if (hist_field)
1499 return hist_field;
1500
1501 list_for_each_entry_rcu(test, &file->triggers, list) {
1502 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1503 test_data = test->private_data;
1504 hist_field = find_var_field(test_data, var_name);
1505 if (hist_field)
1506 return hist_field;
1507 }
1508 }
1509
1510 return NULL;
1511}
1512
067fe038
TZ		 1513static struct trace_event_file *find_var_file(struct trace_array *tr,
1514 char *system,
1515 char *event_name,
1516 char *var_name)
1517{
1518 struct hist_trigger_data *var_hist_data;
1519 struct hist_var_data *var_data;
1520 struct trace_event_file *file, *found = NULL;
1521
1522 if (system)
1523 return find_event_file(tr, system, event_name);
1524
1525 list_for_each_entry(var_data, &tr->hist_vars, list) {
1526 var_hist_data = var_data->hist_data;
1527 file = var_hist_data->event_file;
1528 if (file == found)
1529 continue;
1530
1531 if (find_var_field(var_hist_data, var_name)) {
f404da6e
TZ		 1532 if (found) {
1533 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
067fe038 1534 return NULL;
f404da6e 1535 }
067fe038
TZ		 1536
1537 found = file;
1538 }
1539 }
1540
1541 return found;
1542}
1543
1544static struct hist_field *find_file_var(struct trace_event_file *file,
1545 const char *var_name)
1546{
1547 struct hist_trigger_data *test_data;
1548 struct event_trigger_data *test;
1549 struct hist_field *hist_field;
1550
1551 list_for_each_entry_rcu(test, &file->triggers, list) {
1552 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1553 test_data = test->private_data;
1554 hist_field = find_var_field(test_data, var_name);
1555 if (hist_field)
1556 return hist_field;
1557 }
1558 }
1559
1560 return NULL;
1561}
1562
c282a386
TZ		 1563static struct hist_field *
1564find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1565{
1566 struct trace_array *tr = hist_data->event_file->tr;
1567 struct hist_field *hist_field, *found = NULL;
1568 struct trace_event_file *file;
1569 unsigned int i;
1570
1571 for (i = 0; i < hist_data->n_actions; i++) {
1572 struct action_data *data = hist_data->actions[i];
1573
1574 if (data->fn == action_trace) {
1575 char *system = data->onmatch.match_event_system;
1576 char *event_name = data->onmatch.match_event;
1577
1578 file = find_var_file(tr, system, event_name, var_name);
1579 if (!file)
1580 continue;
1581 hist_field = find_file_var(file, var_name);
1582 if (hist_field) {
1583 if (found) {
f404da6e 1584 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
c282a386
TZ		 1585 return ERR_PTR(-EINVAL);
1586 }
1587
1588 found = hist_field;
1589 }
1590 }
1591 }
1592 return found;
1593}
1594
067fe038
TZ		 1595static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1596 char *system,
1597 char *event_name,
1598 char *var_name)
1599{
1600 struct trace_array *tr = hist_data->event_file->tr;
1601 struct hist_field *hist_field = NULL;
1602 struct trace_event_file *file;
1603
c282a386
TZ		 1604 if (!system || !event_name) {
1605 hist_field = find_match_var(hist_data, var_name);
1606 if (IS_ERR(hist_field))
1607 return NULL;
1608 if (hist_field)
1609 return hist_field;
1610 }
1611
067fe038
TZ		 1612 file = find_var_file(tr, system, event_name, var_name);
1613 if (!file)
1614 return NULL;
1615
1616 hist_field = find_file_var(file, var_name);
1617
1618 return hist_field;
1619}
1620
af6a29bc
TZ		 1621struct hist_elt_data {
1622 char *comm;
067fe038 1623 u64 *var_ref_vals;
02205a67 1624 char *field_var_str[SYNTH_FIELDS_MAX];
af6a29bc
TZ		 1625};
1626
067fe038
TZ		 1627static u64 hist_field_var_ref(struct hist_field *hist_field,
1628 struct tracing_map_elt *elt,
1629 struct ring_buffer_event *rbe,
1630 void *event)
1631{
1632 struct hist_elt_data *elt_data;
1633 u64 var_val = 0;
1634
86895090
TZ		 1635 if (WARN_ON_ONCE(!elt))
1636 return var_val;
1637
067fe038
TZ		 1638 elt_data = elt->private_data;
1639 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1640
1641 return var_val;
1642}
1643
1644static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1645 u64 *var_ref_vals, bool self)
1646{
1647 struct hist_trigger_data *var_data;
1648 struct tracing_map_elt *var_elt;
1649 struct hist_field *hist_field;
1650 unsigned int i, var_idx;
1651 bool resolved = true;
1652 u64 var_val = 0;
1653
1654 for (i = 0; i < hist_data->n_var_refs; i++) {
1655 hist_field = hist_data->var_refs[i];
1656 var_idx = hist_field->var.idx;
1657 var_data = hist_field->var.hist_data;
1658
1659 if (var_data == NULL) {
1660 resolved = false;
1661 break;
1662 }
1663
1664 if ((self && var_data != hist_data) ||
1665 (!self && var_data == hist_data))
1666 continue;
1667
1668 var_elt = tracing_map_lookup(var_data->map, key);
1669 if (!var_elt) {
1670 resolved = false;
1671 break;
1672 }
1673
1674 if (!tracing_map_var_set(var_elt, var_idx)) {
1675 resolved = false;
1676 break;
1677 }
1678
1679 if (self || !hist_field->read_once)
1680 var_val = tracing_map_read_var(var_elt, var_idx);
1681 else
1682 var_val = tracing_map_read_var_once(var_elt, var_idx);
1683
1684 var_ref_vals[i] = var_val;
1685 }
1686
1687 return resolved;
1688}
1689
85013256
TZ		 1690static const char *hist_field_name(struct hist_field *field,
1691 unsigned int level)
1692{
1693 const char *field_name = "";
1694
1695 if (level > 1)
1696 return field_name;
1697
1698 if (field->field)
1699 field_name = field->field->name;
7e8b88a3
TZ		 1700 else if (field->flags & HIST_FIELD_FL_LOG2 ||
1701 field->flags & HIST_FIELD_FL_ALIAS)
5819eadd 1702 field_name = hist_field_name(field->operands[0], ++level);
8b7622bf
TZ		 1703 else if (field->flags & HIST_FIELD_FL_CPU)
1704 field_name = "cpu";
067fe038
TZ		 1705 else if (field->flags & HIST_FIELD_FL_EXPR ||
1706 field->flags & HIST_FIELD_FL_VAR_REF) {
1707 if (field->system) {
1708 static char full_name[MAX_FILTER_STR_VAL];
1709
1710 strcat(full_name, field->system);
1711 strcat(full_name, ".");
1712 strcat(full_name, field->event_name);
1713 strcat(full_name, ".");
1714 strcat(full_name, field->name);
1715 field_name = full_name;
1716 } else
1717 field_name = field->name;
0ae7961e
TZ		 1718 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1719 field_name = "common_timestamp";
85013256
TZ		 1720
1721 if (field_name == NULL)
1722 field_name = "";
1723
1724 return field_name;
1725}
1726
7ef224d1
TZ		 1727static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1728{
1729 hist_field_fn_t fn = NULL;
1730
1731 switch (field_size) {
1732 case 8:
1733 if (field_is_signed)
1734 fn = hist_field_s64;
1735 else
1736 fn = hist_field_u64;
1737 break;
1738 case 4:
1739 if (field_is_signed)
1740 fn = hist_field_s32;
1741 else
1742 fn = hist_field_u32;
1743 break;
1744 case 2:
1745 if (field_is_signed)
1746 fn = hist_field_s16;
1747 else
1748 fn = hist_field_u16;
1749 break;
1750 case 1:
1751 if (field_is_signed)
1752 fn = hist_field_s8;
1753 else
1754 fn = hist_field_u8;
1755 break;
1756 }
1757
1758 return fn;
1759}
1760
1761static int parse_map_size(char *str)
1762{
1763 unsigned long size, map_bits;
1764 int ret;
1765
1766 strsep(&str, "=");
1767 if (!str) {
1768 ret = -EINVAL;
1769 goto out;
1770 }
1771
1772 ret = kstrtoul(str, 0, &size);
1773 if (ret)
1774 goto out;
1775
1776 map_bits = ilog2(roundup_pow_of_two(size));
1777 if (map_bits < TRACING_MAP_BITS_MIN ||
1778 map_bits > TRACING_MAP_BITS_MAX)
1779 ret = -EINVAL;
1780 else
1781 ret = map_bits;
1782 out:
1783 return ret;
1784}
1785
1786static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1787{
30350d65
TZ		 1788 unsigned int i;
1789
7ef224d1
TZ		 1790 if (!attrs)
1791 return;
1792
30350d65
TZ		 1793 for (i = 0; i < attrs->n_assignments; i++)
1794 kfree(attrs->assignment_str[i]);
1795
0212e2aa
TZ		 1796 for (i = 0; i < attrs->n_actions; i++)
1797 kfree(attrs->action_str[i]);
1798
5463bfda 1799 kfree(attrs->name);
e62347d2 1800 kfree(attrs->sort_key_str);
7ef224d1 1801 kfree(attrs->keys_str);
f2606835 1802 kfree(attrs->vals_str);
a4072fe8 1803 kfree(attrs->clock);
7ef224d1
TZ		 1804 kfree(attrs);
1805}
1806
0212e2aa
TZ		 1807static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1808{
c282a386 1809 int ret = -EINVAL;
0212e2aa
TZ		 1810
1811 if (attrs->n_actions >= HIST_ACTIONS_MAX)
1812 return ret;
1813
50450603
TZ		 1814 if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) ||
1815 (strncmp(str, "onmax(", strlen("onmax(")) == 0)) {
c282a386
TZ		 1816 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1817 if (!attrs->action_str[attrs->n_actions]) {
1818 ret = -ENOMEM;
1819 return ret;
1820 }
1821 attrs->n_actions++;
1822 ret = 0;
1823 }
1824
0212e2aa
TZ		 1825 return ret;
1826}
1827
9b1ae035
TZ		 1828static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1829{
1830 int ret = 0;
1831
1832 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
1833 (strncmp(str, "keys=", strlen("keys=")) == 0)) {
1834 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1835 if (!attrs->keys_str) {
1836 ret = -ENOMEM;
1837 goto out;
1838 }
1839 } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
1840 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
1841 (strncmp(str, "values=", strlen("values=")) == 0)) {
1842 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1843 if (!attrs->vals_str) {
1844 ret = -ENOMEM;
1845 goto out;
1846 }
1847 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
1848 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1849 if (!attrs->sort_key_str) {
1850 ret = -ENOMEM;
1851 goto out;
1852 }
1853 } else if (strncmp(str, "name=", strlen("name=")) == 0) {
1854 attrs->name = kstrdup(str, GFP_KERNEL);
1855 if (!attrs->name) {
1856 ret = -ENOMEM;
1857 goto out;
1858 }
a4072fe8
TZ		 1859 } else if (strncmp(str, "clock=", strlen("clock=")) == 0) {
1860 strsep(&str, "=");
1861 if (!str) {
1862 ret = -EINVAL;
1863 goto out;
1864 }
1865
1866 str = strstrip(str);
1867 attrs->clock = kstrdup(str, GFP_KERNEL);
1868 if (!attrs->clock) {
1869 ret = -ENOMEM;
1870 goto out;
1871 }
9b1ae035
TZ		 1872 } else if (strncmp(str, "size=", strlen("size=")) == 0) {
1873 int map_bits = parse_map_size(str);
1874
1875 if (map_bits < 0) {
1876 ret = map_bits;
1877 goto out;
1878 }
1879 attrs->map_bits = map_bits;
30350d65
TZ		 1880 } else {
1881 char *assignment;
1882
1883 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
f404da6e 1884 hist_err("Too many variables defined: ", str);
30350d65
TZ		 1885 ret = -EINVAL;
1886 goto out;
1887 }
1888
1889 assignment = kstrdup(str, GFP_KERNEL);
1890 if (!assignment) {
1891 ret = -ENOMEM;
1892 goto out;
1893 }
1894
1895 attrs->assignment_str[attrs->n_assignments++] = assignment;
1896 }
9b1ae035
TZ		 1897 out:
1898 return ret;
1899}
1900
7ef224d1
TZ		 1901static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1902{
1903 struct hist_trigger_attrs *attrs;
1904 int ret = 0;
1905
1906 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1907 if (!attrs)
1908 return ERR_PTR(-ENOMEM);
1909
1910 while (trigger_str) {
1911 char *str = strsep(&trigger_str, ":");
1912
9b1ae035
TZ		 1913 if (strchr(str, '=')) {
1914 ret = parse_assignment(str, attrs);
1915 if (ret)
1916 goto free;
1917 } else if (strcmp(str, "pause") == 0)
83e99914
TZ		 1918 attrs->pause = true;
1919 else if ((strcmp(str, "cont") == 0) ||
1920 (strcmp(str, "continue") == 0))
1921 attrs->cont = true;
e86ae9ba
TZ		 1922 else if (strcmp(str, "clear") == 0)
1923 attrs->clear = true;
9b1ae035 1924 else {
0212e2aa
TZ		 1925 ret = parse_action(str, attrs);
1926 if (ret)
1927 goto free;
7ef224d1
TZ		 1928 }
1929 }
1930
1931 if (!attrs->keys_str) {
1932 ret = -EINVAL;
1933 goto free;
1934 }
1935
a4072fe8
TZ		 1936 if (!attrs->clock) {
1937 attrs->clock = kstrdup("global", GFP_KERNEL);
1938 if (!attrs->clock) {
1939 ret = -ENOMEM;
1940 goto free;
1941 }
1942 }
1943
7ef224d1
TZ		 1944 return attrs;
1945 free:
1946 destroy_hist_trigger_attrs(attrs);
1947
1948 return ERR_PTR(ret);
1949}
1950
6b4827ad
TZ		 1951static inline void save_comm(char *comm, struct task_struct *task)
1952{
1953 if (!task->pid) {
1954 strcpy(comm, "<idle>");
1955 return;
1956 }
1957
1958 if (WARN_ON_ONCE(task->pid < 0)) {
1959 strcpy(comm, "<XXX>");
1960 return;
1961 }
1962
1963 memcpy(comm, task->comm, TASK_COMM_LEN);
1964}
1965
af6a29bc
TZ		 1966static void hist_elt_data_free(struct hist_elt_data *elt_data)
1967{
02205a67
TZ		 1968 unsigned int i;
1969
1970 for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1971 kfree(elt_data->field_var_str[i]);
1972
af6a29bc
TZ		 1973 kfree(elt_data->comm);
1974 kfree(elt_data);
1975}
1976
1977static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
6b4827ad 1978{
af6a29bc
TZ		 1979 struct hist_elt_data *elt_data = elt->private_data;
1980
1981 hist_elt_data_free(elt_data);
6b4827ad
TZ		 1982}
1983
af6a29bc 1984static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
6b4827ad
TZ		 1985{
1986 struct hist_trigger_data *hist_data = elt->map->private_data;
af6a29bc
TZ		 1987 unsigned int size = TASK_COMM_LEN;
1988 struct hist_elt_data *elt_data;
6b4827ad 1989 struct hist_field *key_field;
02205a67 1990 unsigned int i, n_str;
6b4827ad 1991
af6a29bc
TZ		 1992 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1993 if (!elt_data)
1994 return -ENOMEM;
1995
6b4827ad
TZ		 1996 for_each_hist_key_field(i, hist_data) {
1997 key_field = hist_data->fields[i];
1998
1999 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 2000 elt_data->comm = kzalloc(size, GFP_KERNEL);
2001 if (!elt_data->comm) {
2002 kfree(elt_data);
6b4827ad 2003 return -ENOMEM;
af6a29bc 2004 }
6b4827ad
TZ		 2005 break;
2006 }
2007 }
2008
50450603 2009 n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
02205a67
TZ		 2010
2011 size = STR_VAR_LEN_MAX;
2012
2013 for (i = 0; i < n_str; i++) {
2014 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2015 if (!elt_data->field_var_str[i]) {
2016 hist_elt_data_free(elt_data);
2017 return -ENOMEM;
2018 }
2019 }
2020
af6a29bc
TZ		 2021 elt->private_data = elt_data;
2022
6b4827ad
TZ		 2023 return 0;
2024}
2025
af6a29bc 2026static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
6b4827ad 2027{
af6a29bc 2028 struct hist_elt_data *elt_data = elt->private_data;
6b4827ad 2029
af6a29bc
TZ		 2030 if (elt_data->comm)
2031 save_comm(elt_data->comm, current);
6b4827ad
TZ		 2032}
2033
af6a29bc
TZ		 2034static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2035 .elt_alloc = hist_trigger_elt_data_alloc,
2036 .elt_free = hist_trigger_elt_data_free,
2037 .elt_init = hist_trigger_elt_data_init,
6b4827ad
TZ		 2038};
2039
2ece94fb
TZ		 2040static const char *get_hist_field_flags(struct hist_field *hist_field)
2041{
2042 const char *flags_str = NULL;
2043
2044 if (hist_field->flags & HIST_FIELD_FL_HEX)
2045 flags_str = "hex";
2046 else if (hist_field->flags & HIST_FIELD_FL_SYM)
2047 flags_str = "sym";
2048 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2049 flags_str = "sym-offset";
2050 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2051 flags_str = "execname";
2052 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2053 flags_str = "syscall";
2054 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2055 flags_str = "log2";
2056 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2057 flags_str = "usecs";
2058
2059 return flags_str;
2060}
2061
100719dc
TZ		 2062static void expr_field_str(struct hist_field *field, char *expr)
2063{
067fe038
TZ		 2064 if (field->flags & HIST_FIELD_FL_VAR_REF)
2065 strcat(expr, "$");
2066
100719dc
TZ		 2067 strcat(expr, hist_field_name(field, 0));
2068
76690945 2069 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
100719dc
TZ		 2070 const char *flags_str = get_hist_field_flags(field);
2071
2072 if (flags_str) {
2073 strcat(expr, ".");
2074 strcat(expr, flags_str);
2075 }
2076 }
2077}
2078
2079static char *expr_str(struct hist_field *field, unsigned int level)
2080{
2081 char *expr;
2082
2083 if (level > 1)
2084 return NULL;
2085
2086 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2087 if (!expr)
2088 return NULL;
2089
2090 if (!field->operands[0]) {
2091 expr_field_str(field, expr);
2092 return expr;
2093 }
2094
2095 if (field->operator == FIELD_OP_UNARY_MINUS) {
2096 char *subexpr;
2097
2098 strcat(expr, "-(");
2099 subexpr = expr_str(field->operands[0], ++level);
2100 if (!subexpr) {
2101 kfree(expr);
2102 return NULL;
2103 }
2104 strcat(expr, subexpr);
2105 strcat(expr, ")");
2106
2107 kfree(subexpr);
2108
2109 return expr;
2110 }
2111
2112 expr_field_str(field->operands[0], expr);
2113
2114 switch (field->operator) {
2115 case FIELD_OP_MINUS:
2116 strcat(expr, "-");
2117 break;
2118 case FIELD_OP_PLUS:
2119 strcat(expr, "+");
2120 break;
2121 default:
2122 kfree(expr);
2123 return NULL;
2124 }
2125
2126 expr_field_str(field->operands[1], expr);
2127
2128 return expr;
2129}
2130
2131static int contains_operator(char *str)
2132{
2133 enum field_op_id field_op = FIELD_OP_NONE;
2134 char *op;
2135
2136 op = strpbrk(str, "+-");
2137 if (!op)
2138 return FIELD_OP_NONE;
2139
2140 switch (*op) {
2141 case '-':
2142 if (*str == '-')
2143 field_op = FIELD_OP_UNARY_MINUS;
2144 else
2145 field_op = FIELD_OP_MINUS;
2146 break;
2147 case '+':
2148 field_op = FIELD_OP_PLUS;
2149 break;
2150 default:
2151 break;
2152 }
2153
2154 return field_op;
2155}
2156
5819eadd
TZ		 2157static void destroy_hist_field(struct hist_field *hist_field,
2158 unsigned int level)
7ef224d1 2159{
5819eadd
TZ		 2160 unsigned int i;
2161
100719dc 2162 if (level > 3)
5819eadd
TZ		 2163 return;
2164
2165 if (!hist_field)
2166 return;
2167
2168 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2169 destroy_hist_field(hist_field->operands[i], level + 1);
2170
30350d65 2171 kfree(hist_field->var.name);
100719dc 2172 kfree(hist_field->name);
19a9facd 2173 kfree(hist_field->type);
30350d65 2174
7ef224d1
TZ		 2175 kfree(hist_field);
2176}
2177
b559d003
TZ		 2178static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2179 struct ftrace_event_field *field,
30350d65
TZ		 2180 unsigned long flags,
2181 char *var_name)
7ef224d1
TZ		 2182{
2183 struct hist_field *hist_field;
2184
2185 if (field && is_function_field(field))
2186 return NULL;
2187
2188 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2189 if (!hist_field)
2190 return NULL;
2191
b559d003
TZ		 2192 hist_field->hist_data = hist_data;
2193
7e8b88a3 2194 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
100719dc
TZ		 2195 goto out; /* caller will populate */
2196
067fe038
TZ		 2197 if (flags & HIST_FIELD_FL_VAR_REF) {
2198 hist_field->fn = hist_field_var_ref;
2199 goto out;
2200 }
2201
7ef224d1
TZ		 2202 if (flags & HIST_FIELD_FL_HITCOUNT) {
2203 hist_field->fn = hist_field_counter;
19a9facd
TZ		 2204 hist_field->size = sizeof(u64);
2205 hist_field->type = kstrdup("u64", GFP_KERNEL);
2206 if (!hist_field->type)
2207 goto free;
7ef224d1
TZ		 2208 goto out;
2209 }
2210
69a0200c
TZ		 2211 if (flags & HIST_FIELD_FL_STACKTRACE) {
2212 hist_field->fn = hist_field_none;
2213 goto out;
2214 }
2215
4b94f5b7 2216 if (flags & HIST_FIELD_FL_LOG2) {
5819eadd 2217 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
4b94f5b7 2218 hist_field->fn = hist_field_log2;
30350d65 2219 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
5819eadd 2220 hist_field->size = hist_field->operands[0]->size;
19a9facd
TZ		 2221 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2222 if (!hist_field->type)
2223 goto free;
4b94f5b7
NK		 2224 goto out;
2225 }
2226
ad42febe
TZ		 2227 if (flags & HIST_FIELD_FL_TIMESTAMP) {
2228 hist_field->fn = hist_field_timestamp;
2229 hist_field->size = sizeof(u64);
19a9facd
TZ		 2230 hist_field->type = kstrdup("u64", GFP_KERNEL);
2231 if (!hist_field->type)
2232 goto free;
ad42febe
TZ		 2233 goto out;
2234 }
2235
8b7622bf
TZ		 2236 if (flags & HIST_FIELD_FL_CPU) {
2237 hist_field->fn = hist_field_cpu;
2238 hist_field->size = sizeof(int);
2239 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2240 if (!hist_field->type)
2241 goto free;
2242 goto out;
2243 }
2244
432480c5
TZ		 2245 if (WARN_ON_ONCE(!field))
2246 goto out;
2247
7ef224d1
TZ		 2248 if (is_string_field(field)) {
2249 flags |= HIST_FIELD_FL_STRING;
79e577cb 2250
19a9facd
TZ		 2251 hist_field->size = MAX_FILTER_STR_VAL;
2252 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2253 if (!hist_field->type)
2254 goto free;
2255
79e577cb
NK		 2256 if (field->filter_type == FILTER_STATIC_STRING)
2257 hist_field->fn = hist_field_string;
2258 else if (field->filter_type == FILTER_DYN_STRING)
2259 hist_field->fn = hist_field_dynstring;
2260 else
2261 hist_field->fn = hist_field_pstring;
7ef224d1 2262 } else {
19a9facd
TZ		 2263 hist_field->size = field->size;
2264 hist_field->is_signed = field->is_signed;
2265 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2266 if (!hist_field->type)
2267 goto free;
2268
7ef224d1
TZ		 2269 hist_field->fn = select_value_fn(field->size,
2270 field->is_signed);
2271 if (!hist_field->fn) {
5819eadd 2272 destroy_hist_field(hist_field, 0);
7ef224d1
TZ		 2273 return NULL;
2274 }
2275 }
2276 out:
2277 hist_field->field = field;
2278 hist_field->flags = flags;
2279
30350d65
TZ		 2280 if (var_name) {
2281 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2282 if (!hist_field->var.name)
2283 goto free;
2284 }
2285
7ef224d1 2286 return hist_field;
30350d65
TZ		 2287 free:
2288 destroy_hist_field(hist_field, 0);
2289 return NULL;
7ef224d1
TZ		 2290}
2291
2292static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2293{
2294 unsigned int i;
2295
30350d65 2296 for (i = 0; i < HIST_FIELDS_MAX; i++) {
7ef224d1 2297 if (hist_data->fields[i]) {
5819eadd 2298 destroy_hist_field(hist_data->fields[i], 0);
7ef224d1
TZ		 2299 hist_data->fields[i] = NULL;
2300 }
2301 }
2302}
2303
067fe038
TZ		 2304static int init_var_ref(struct hist_field *ref_field,
2305 struct hist_field *var_field,
2306 char *system, char *event_name)
2307{
2308 int err = 0;
2309
2310 ref_field->var.idx = var_field->var.idx;
2311 ref_field->var.hist_data = var_field->hist_data;
2312 ref_field->size = var_field->size;
2313 ref_field->is_signed = var_field->is_signed;
2314 ref_field->flags |= var_field->flags &
2315 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2316
2317 if (system) {
2318 ref_field->system = kstrdup(system, GFP_KERNEL);
2319 if (!ref_field->system)
2320 return -ENOMEM;
2321 }
2322
2323 if (event_name) {
2324 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2325 if (!ref_field->event_name) {
2326 err = -ENOMEM;
2327 goto free;
2328 }
2329 }
2330
7e8b88a3
TZ		 2331 if (var_field->var.name) {
2332 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2333 if (!ref_field->name) {
2334 err = -ENOMEM;
2335 goto free;
2336 }
2337 } else if (var_field->name) {
2338 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2339 if (!ref_field->name) {
2340 err = -ENOMEM;
2341 goto free;
2342 }
067fe038
TZ		 2343 }
2344
2345 ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2346 if (!ref_field->type) {
2347 err = -ENOMEM;
2348 goto free;
2349 }
2350 out:
2351 return err;
2352 free:
2353 kfree(ref_field->system);
2354 kfree(ref_field->event_name);
2355 kfree(ref_field->name);
2356
2357 goto out;
2358}
2359
2360static struct hist_field *create_var_ref(struct hist_field *var_field,
2361 char *system, char *event_name)
2362{
2363 unsigned long flags = HIST_FIELD_FL_VAR_REF;
2364 struct hist_field *ref_field;
2365
2366 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2367 if (ref_field) {
2368 if (init_var_ref(ref_field, var_field, system, event_name)) {
2369 destroy_hist_field(ref_field, 0);
2370 return NULL;
2371 }
2372 }
2373
2374 return ref_field;
2375}
2376
2377static bool is_var_ref(char *var_name)
2378{
2379 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2380 return false;
2381
2382 return true;
2383}
2384
2385static char *field_name_from_var(struct hist_trigger_data *hist_data,
2386 char *var_name)
2387{
2388 char *name, *field;
2389 unsigned int i;
2390
2391 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2392 name = hist_data->attrs->var_defs.name[i];
2393
2394 if (strcmp(var_name, name) == 0) {
2395 field = hist_data->attrs->var_defs.expr[i];
2396 if (contains_operator(field) || is_var_ref(field))
2397 continue;
2398 return field;
2399 }
2400 }
2401
2402 return NULL;
2403}
2404
2405static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2406 char *system, char *event_name,
2407 char *var_name)
2408{
2409 struct trace_event_call *call;
2410
2411 if (system && event_name) {
2412 call = hist_data->event_file->event_call;
2413
2414 if (strcmp(system, call->class->system) != 0)
2415 return NULL;
2416
2417 if (strcmp(event_name, trace_event_name(call)) != 0)
2418 return NULL;
2419 }
2420
2421 if (!!system != !!event_name)
2422 return NULL;
2423
2424 if (!is_var_ref(var_name))
2425 return NULL;
2426
2427 var_name++;
2428
2429 return field_name_from_var(hist_data, var_name);
2430}
2431
2432static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2433 char *system, char *event_name,
2434 char *var_name)
2435{
2436 struct hist_field *var_field = NULL, *ref_field = NULL;
2437
2438 if (!is_var_ref(var_name))
2439 return NULL;
2440
2441 var_name++;
2442
2443 var_field = find_event_var(hist_data, system, event_name, var_name);
2444 if (var_field)
2445 ref_field = create_var_ref(var_field, system, event_name);
2446
f404da6e
TZ		 2447 if (!ref_field)
2448 hist_err_event("Couldn't find variable: $",
2449 system, event_name, var_name);
2450
067fe038
TZ		 2451 return ref_field;
2452}
2453
100719dc
TZ		 2454static struct ftrace_event_field *
2455parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2456 char *field_str, unsigned long *flags)
2457{
2458 struct ftrace_event_field *field = NULL;
2459 char *field_name, *modifier, *str;
2460
2461 modifier = str = kstrdup(field_str, GFP_KERNEL);
2462 if (!modifier)
2463 return ERR_PTR(-ENOMEM);
2464
2465 field_name = strsep(&modifier, ".");
2466 if (modifier) {
2467 if (strcmp(modifier, "hex") == 0)
2468 *flags |= HIST_FIELD_FL_HEX;
2469 else if (strcmp(modifier, "sym") == 0)
2470 *flags |= HIST_FIELD_FL_SYM;
2471 else if (strcmp(modifier, "sym-offset") == 0)
2472 *flags |= HIST_FIELD_FL_SYM_OFFSET;
2473 else if ((strcmp(modifier, "execname") == 0) &&
2474 (strcmp(field_name, "common_pid") == 0))
2475 *flags |= HIST_FIELD_FL_EXECNAME;
2476 else if (strcmp(modifier, "syscall") == 0)
2477 *flags |= HIST_FIELD_FL_SYSCALL;
2478 else if (strcmp(modifier, "log2") == 0)
2479 *flags |= HIST_FIELD_FL_LOG2;
2480 else if (strcmp(modifier, "usecs") == 0)
2481 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2482 else {
dcf23457 2483 hist_err("Invalid field modifier: ", modifier);
100719dc
TZ		 2484 field = ERR_PTR(-EINVAL);
2485 goto out;
2486 }
2487 }
2488
2489 if (strcmp(field_name, "common_timestamp") == 0) {
2490 *flags |= HIST_FIELD_FL_TIMESTAMP;
2491 hist_data->enable_timestamps = true;
2492 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2493 hist_data->attrs->ts_in_usecs = true;
8b7622bf
TZ		 2494 } else if (strcmp(field_name, "cpu") == 0)
2495 *flags |= HIST_FIELD_FL_CPU;
2496 else {
100719dc
TZ		 2497 field = trace_find_event_field(file->event_call, field_name);
2498 if (!field || !field->size) {
5ec432d7 2499 hist_err("Couldn't find field: ", field_name);
100719dc
TZ		 2500 field = ERR_PTR(-EINVAL);
2501 goto out;
2502 }
2503 }
2504 out:
2505 kfree(str);
2506
2507 return field;
2508}
2509
7e8b88a3
TZ		 2510static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2511 struct hist_field *var_ref,
2512 char *var_name)
2513{
2514 struct hist_field *alias = NULL;
2515 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2516
2517 alias = create_hist_field(hist_data, NULL, flags, var_name);
2518 if (!alias)
2519 return NULL;
2520
2521 alias->fn = var_ref->fn;
2522 alias->operands[0] = var_ref;
2523
2524 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2525 destroy_hist_field(alias, 0);
2526 return NULL;
2527 }
2528
2529 return alias;
2530}
2531
100719dc
TZ		 2532static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2533 struct trace_event_file *file, char *str,
2534 unsigned long *flags, char *var_name)
2535{
067fe038 2536 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
100719dc
TZ		 2537 struct ftrace_event_field *field = NULL;
2538 struct hist_field *hist_field = NULL;
2539 int ret = 0;
2540
067fe038
TZ		 2541 s = strchr(str, '.');
2542 if (s) {
2543 s = strchr(++s, '.');
2544 if (s) {
2545 ref_system = strsep(&str, ".");
2546 if (!str) {
2547 ret = -EINVAL;
2548 goto out;
2549 }
2550 ref_event = strsep(&str, ".");
2551 if (!str) {
2552 ret = -EINVAL;
2553 goto out;
2554 }
2555 ref_var = str;
2556 }
2557 }
2558
2559 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2560 if (!s) {
2561 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2562 if (hist_field) {
2563 hist_data->var_refs[hist_data->n_var_refs] = hist_field;
2564 hist_field->var_ref_idx = hist_data->n_var_refs++;
7e8b88a3
TZ		 2565 if (var_name) {
2566 hist_field = create_alias(hist_data, hist_field, var_name);
2567 if (!hist_field) {
2568 ret = -ENOMEM;
2569 goto out;
2570 }
2571 }
067fe038
TZ		 2572 return hist_field;
2573 }
2574 } else
2575 str = s;
2576
100719dc
TZ		 2577 field = parse_field(hist_data, file, str, flags);
2578 if (IS_ERR(field)) {
2579 ret = PTR_ERR(field);
2580 goto out;
2581 }
2582
2583 hist_field = create_hist_field(hist_data, field, *flags, var_name);
2584 if (!hist_field) {
2585 ret = -ENOMEM;
2586 goto out;
2587 }
2588
2589 return hist_field;
2590 out:
2591 return ERR_PTR(ret);
2592}
2593
2594static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2595 struct trace_event_file *file,
2596 char *str, unsigned long flags,
2597 char *var_name, unsigned int level);
2598
2599static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2600 struct trace_event_file *file,
2601 char *str, unsigned long flags,
2602 char *var_name, unsigned int level)
2603{
2604 struct hist_field *operand1, *expr = NULL;
2605 unsigned long operand_flags;
2606 int ret = 0;
2607 char *s;
2608
2609 /* we support only -(xxx) i.e. explicit parens required */
2610
2611 if (level > 3) {
f404da6e 2612 hist_err("Too many subexpressions (3 max): ", str);
100719dc
TZ		 2613 ret = -EINVAL;
2614 goto free;
2615 }
2616
2617 str++; /* skip leading '-' */
2618
2619 s = strchr(str, '(');
2620 if (s)
2621 str++;
2622 else {
2623 ret = -EINVAL;
2624 goto free;
2625 }
2626
2627 s = strrchr(str, ')');
2628 if (s)
2629 *s = '\0';
2630 else {
2631 ret = -EINVAL; /* no closing ')' */
2632 goto free;
2633 }
2634
2635 flags |= HIST_FIELD_FL_EXPR;
2636 expr = create_hist_field(hist_data, NULL, flags, var_name);
2637 if (!expr) {
2638 ret = -ENOMEM;
2639 goto free;
2640 }
2641
2642 operand_flags = 0;
2643 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2644 if (IS_ERR(operand1)) {
2645 ret = PTR_ERR(operand1);
2646 goto free;
2647 }
2648
2649 expr->flags |= operand1->flags &
2650 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2651 expr->fn = hist_field_unary_minus;
2652 expr->operands[0] = operand1;
2653 expr->operator = FIELD_OP_UNARY_MINUS;
2654 expr->name = expr_str(expr, 0);
19a9facd
TZ		 2655 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2656 if (!expr->type) {
2657 ret = -ENOMEM;
2658 goto free;
2659 }
100719dc
TZ		 2660
2661 return expr;
2662 free:
2663 destroy_hist_field(expr, 0);
2664 return ERR_PTR(ret);
2665}
2666
2667static int check_expr_operands(struct hist_field *operand1,
2668 struct hist_field *operand2)
2669{
2670 unsigned long operand1_flags = operand1->flags;
2671 unsigned long operand2_flags = operand2->flags;
2672
7e8b88a3
TZ		 2673 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2674 (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2675 struct hist_field *var;
2676
2677 var = find_var_field(operand1->var.hist_data, operand1->name);
2678 if (!var)
2679 return -EINVAL;
2680 operand1_flags = var->flags;
2681 }
2682
2683 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2684 (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2685 struct hist_field *var;
2686
2687 var = find_var_field(operand2->var.hist_data, operand2->name);
2688 if (!var)
2689 return -EINVAL;
2690 operand2_flags = var->flags;
2691 }
2692
100719dc 2693 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
f404da6e
TZ		 2694 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2695 hist_err("Timestamp units in expression don't match", NULL);
100719dc 2696 return -EINVAL;
f404da6e 2697 }
100719dc
TZ		 2698
2699 return 0;
2700}
2701
2702static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2703 struct trace_event_file *file,
2704 char *str, unsigned long flags,
2705 char *var_name, unsigned int level)
2706{
2707 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2708 unsigned long operand_flags;
2709 int field_op, ret = -EINVAL;
2710 char *sep, *operand1_str;
2711
f404da6e
TZ		 2712 if (level > 3) {
2713 hist_err("Too many subexpressions (3 max): ", str);
100719dc 2714 return ERR_PTR(-EINVAL);
f404da6e 2715 }
100719dc
TZ		 2716
2717 field_op = contains_operator(str);
2718
2719 if (field_op == FIELD_OP_NONE)
2720 return parse_atom(hist_data, file, str, &flags, var_name);
2721
2722 if (field_op == FIELD_OP_UNARY_MINUS)
2723 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2724
2725 switch (field_op) {
2726 case FIELD_OP_MINUS:
2727 sep = "-";
2728 break;
2729 case FIELD_OP_PLUS:
2730 sep = "+";
2731 break;
2732 default:
2733 goto free;
2734 }
2735
2736 operand1_str = strsep(&str, sep);
2737 if (!operand1_str || !str)
2738 goto free;
2739
2740 operand_flags = 0;
2741 operand1 = parse_atom(hist_data, file, operand1_str,
2742 &operand_flags, NULL);
2743 if (IS_ERR(operand1)) {
2744 ret = PTR_ERR(operand1);
2745 operand1 = NULL;
2746 goto free;
2747 }
2748
2749 /* rest of string could be another expression e.g. b+c in a+b+c */
2750 operand_flags = 0;
2751 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2752 if (IS_ERR(operand2)) {
2753 ret = PTR_ERR(operand2);
2754 operand2 = NULL;
2755 goto free;
2756 }
2757
2758 ret = check_expr_operands(operand1, operand2);
2759 if (ret)
2760 goto free;
2761
2762 flags |= HIST_FIELD_FL_EXPR;
2763
2764 flags |= operand1->flags &
2765 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2766
2767 expr = create_hist_field(hist_data, NULL, flags, var_name);
2768 if (!expr) {
2769 ret = -ENOMEM;
2770 goto free;
2771 }
2772
067fe038
TZ		 2773 operand1->read_once = true;
2774 operand2->read_once = true;
2775
100719dc
TZ		 2776 expr->operands[0] = operand1;
2777 expr->operands[1] = operand2;
2778 expr->operator = field_op;
2779 expr->name = expr_str(expr, 0);
19a9facd
TZ		 2780 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2781 if (!expr->type) {
2782 ret = -ENOMEM;
2783 goto free;
2784 }
100719dc
TZ		 2785
2786 switch (field_op) {
2787 case FIELD_OP_MINUS:
2788 expr->fn = hist_field_minus;
2789 break;
2790 case FIELD_OP_PLUS:
2791 expr->fn = hist_field_plus;
2792 break;
2793 default:
5e4cf2bf 2794 ret = -EINVAL;
100719dc
TZ		 2795 goto free;
2796 }
2797
2798 return expr;
2799 free:
2800 destroy_hist_field(operand1, 0);
2801 destroy_hist_field(operand2, 0);
2802 destroy_hist_field(expr, 0);
2803
2804 return ERR_PTR(ret);
2805}
2806
02205a67
TZ		 2807static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2808 struct trace_event_file *file)
2809{
2810 struct event_trigger_data *test;
2811
2812 list_for_each_entry_rcu(test, &file->triggers, list) {
2813 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2814 if (test->private_data == hist_data)
2815 return test->filter_str;
2816 }
2817 }
2818
2819 return NULL;
2820}
2821
2822static struct event_command trigger_hist_cmd;
2823static int event_hist_trigger_func(struct event_command *cmd_ops,
2824 struct trace_event_file *file,
2825 char *glob, char *cmd, char *param);
2826
2827static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2828 struct hist_trigger_data *hist_data,
2829 unsigned int n_keys)
2830{
2831 struct hist_field *target_hist_field, *hist_field;
2832 unsigned int n, i, j;
2833
2834 if (hist_data->n_fields - hist_data->n_vals != n_keys)
2835 return false;
2836
2837 i = hist_data->n_vals;
2838 j = target_hist_data->n_vals;
2839
2840 for (n = 0; n < n_keys; n++) {
2841 hist_field = hist_data->fields[i + n];
2842 target_hist_field = target_hist_data->fields[j + n];
2843
2844 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2845 return false;
2846 if (hist_field->size != target_hist_field->size)
2847 return false;
2848 if (hist_field->is_signed != target_hist_field->is_signed)
2849 return false;
2850 }
2851
2852 return true;
2853}
2854
2855static struct hist_trigger_data *
2856find_compatible_hist(struct hist_trigger_data *target_hist_data,
2857 struct trace_event_file *file)
2858{
2859 struct hist_trigger_data *hist_data;
2860 struct event_trigger_data *test;
2861 unsigned int n_keys;
2862
2863 n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2864
2865 list_for_each_entry_rcu(test, &file->triggers, list) {
2866 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2867 hist_data = test->private_data;
2868
2869 if (compatible_keys(target_hist_data, hist_data, n_keys))
2870 return hist_data;
2871 }
2872 }
2873
2874 return NULL;
2875}
2876
2877static struct trace_event_file *event_file(struct trace_array *tr,
2878 char *system, char *event_name)
2879{
2880 struct trace_event_file *file;
2881
3be4c1e5 2882 file = __find_event_file(tr, system, event_name);
02205a67
TZ		 2883 if (!file)
2884 return ERR_PTR(-EINVAL);
2885
2886 return file;
2887}
2888
2889static struct hist_field *
2890find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2891 char *system, char *event_name, char *field_name)
2892{
2893 struct hist_field *event_var;
2894 char *synthetic_name;
2895
2896 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2897 if (!synthetic_name)
2898 return ERR_PTR(-ENOMEM);
2899
2900 strcpy(synthetic_name, "synthetic_");
2901 strcat(synthetic_name, field_name);
2902
2903 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2904
2905 kfree(synthetic_name);
2906
2907 return event_var;
2908}
2909
2910/**
2911 * create_field_var_hist - Automatically create a histogram and var for a field
2912 * @target_hist_data: The target hist trigger
2913 * @subsys_name: Optional subsystem name
2914 * @event_name: Optional event name
2915 * @field_name: The name of the field (and the resulting variable)
2916 *
2917 * Hist trigger actions fetch data from variables, not directly from
2918 * events. However, for convenience, users are allowed to directly
2919 * specify an event field in an action, which will be automatically
2920 * converted into a variable on their behalf.
2921
2922 * If a user specifies a field on an event that isn't the event the
2923 * histogram currently being defined (the target event histogram), the
2924 * only way that can be accomplished is if a new hist trigger is
2925 * created and the field variable defined on that.
2926 *
2927 * This function creates a new histogram compatible with the target
2928 * event (meaning a histogram with the same key as the target
2929 * histogram), and creates a variable for the specified field, but
2930 * with 'synthetic_' prepended to the variable name in order to avoid
2931 * collision with normal field variables.
2932 *
2933 * Return: The variable created for the field.
2934 */
c282a386 2935static struct hist_field *
02205a67
TZ		 2936create_field_var_hist(struct hist_trigger_data *target_hist_data,
2937 char *subsys_name, char *event_name, char *field_name)
2938{
2939 struct trace_array *tr = target_hist_data->event_file->tr;
2940 struct hist_field *event_var = ERR_PTR(-EINVAL);
2941 struct hist_trigger_data *hist_data;
2942 unsigned int i, n, first = true;
2943 struct field_var_hist *var_hist;
2944 struct trace_event_file *file;
2945 struct hist_field *key_field;
2946 char *saved_filter;
2947 char *cmd;
2948 int ret;
2949
f404da6e
TZ		 2950 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2951 hist_err_event("onmatch: Too many field variables defined: ",
2952 subsys_name, event_name, field_name);
02205a67 2953 return ERR_PTR(-EINVAL);
f404da6e 2954 }
02205a67
TZ		 2955
2956 file = event_file(tr, subsys_name, event_name);
2957
2958 if (IS_ERR(file)) {
f404da6e
TZ		 2959 hist_err_event("onmatch: Event file not found: ",
2960 subsys_name, event_name, field_name);
02205a67
TZ		 2961 ret = PTR_ERR(file);
2962 return ERR_PTR(ret);
2963 }
2964
2965 /*
2966 * Look for a histogram compatible with target. We'll use the
2967 * found histogram specification to create a new matching
2968 * histogram with our variable on it. target_hist_data is not
2969 * yet a registered histogram so we can't use that.
2970 */
2971 hist_data = find_compatible_hist(target_hist_data, file);
f404da6e
TZ		 2972 if (!hist_data) {
2973 hist_err_event("onmatch: Matching event histogram not found: ",
2974 subsys_name, event_name, field_name);
02205a67 2975 return ERR_PTR(-EINVAL);
f404da6e 2976 }
02205a67
TZ		 2977
2978 /* See if a synthetic field variable has already been created */
2979 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2980 event_name, field_name);
2981 if (!IS_ERR_OR_NULL(event_var))
2982 return event_var;
2983
2984 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2985 if (!var_hist)
2986 return ERR_PTR(-ENOMEM);
2987
2988 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2989 if (!cmd) {
2990 kfree(var_hist);
2991 return ERR_PTR(-ENOMEM);
2992 }
2993
2994 /* Use the same keys as the compatible histogram */
2995 strcat(cmd, "keys=");
2996
2997 for_each_hist_key_field(i, hist_data) {
2998 key_field = hist_data->fields[i];
2999 if (!first)
3000 strcat(cmd, ",");
3001 strcat(cmd, key_field->field->name);
3002 first = false;
3003 }
3004
3005 /* Create the synthetic field variable specification */
3006 strcat(cmd, ":synthetic_");
3007 strcat(cmd, field_name);
3008 strcat(cmd, "=");
3009 strcat(cmd, field_name);
3010
3011 /* Use the same filter as the compatible histogram */
3012 saved_filter = find_trigger_filter(hist_data, file);
3013 if (saved_filter) {
3014 strcat(cmd, " if ");
3015 strcat(cmd, saved_filter);
3016 }
3017
3018 var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3019 if (!var_hist->cmd) {
3020 kfree(cmd);
3021 kfree(var_hist);
3022 return ERR_PTR(-ENOMEM);
3023 }
3024
3025 /* Save the compatible histogram information */
3026 var_hist->hist_data = hist_data;
3027
3028 /* Create the new histogram with our variable */
3029 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3030 "", "hist", cmd);
3031 if (ret) {
3032 kfree(cmd);
3033 kfree(var_hist->cmd);
3034 kfree(var_hist);
f404da6e
TZ		 3035 hist_err_event("onmatch: Couldn't create histogram for field: ",
3036 subsys_name, event_name, field_name);
02205a67
TZ		 3037 return ERR_PTR(ret);
3038 }
3039
3040 kfree(cmd);
3041
3042 /* If we can't find the variable, something went wrong */
3043 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3044 event_name, field_name);
3045 if (IS_ERR_OR_NULL(event_var)) {
3046 kfree(var_hist->cmd);
3047 kfree(var_hist);
f404da6e
TZ		 3048 hist_err_event("onmatch: Couldn't find synthetic variable: ",
3049 subsys_name, event_name, field_name);
02205a67
TZ		 3050 return ERR_PTR(-EINVAL);
3051 }
3052
3053 n = target_hist_data->n_field_var_hists;
3054 target_hist_data->field_var_hists[n] = var_hist;
3055 target_hist_data->n_field_var_hists++;
3056
3057 return event_var;
3058}
3059
c282a386 3060static struct hist_field *
02205a67
TZ		 3061find_target_event_var(struct hist_trigger_data *hist_data,
3062 char *subsys_name, char *event_name, char *var_name)
3063{
3064 struct trace_event_file *file = hist_data->event_file;
3065 struct hist_field *hist_field = NULL;
3066
3067 if (subsys_name) {
3068 struct trace_event_call *call;
3069
3070 if (!event_name)
3071 return NULL;
3072
3073 call = file->event_call;
3074
3075 if (strcmp(subsys_name, call->class->system) != 0)
3076 return NULL;
3077
3078 if (strcmp(event_name, trace_event_name(call)) != 0)
3079 return NULL;
3080 }
3081
3082 hist_field = find_var_field(hist_data, var_name);
3083
3084 return hist_field;
3085}
3086
3087static inline void __update_field_vars(struct tracing_map_elt *elt,
3088 struct ring_buffer_event *rbe,
3089 void *rec,
3090 struct field_var **field_vars,
3091 unsigned int n_field_vars,
3092 unsigned int field_var_str_start)
3093{
3094 struct hist_elt_data *elt_data = elt->private_data;
3095 unsigned int i, j, var_idx;
3096 u64 var_val;
3097
3098 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3099 struct field_var *field_var = field_vars[i];
3100 struct hist_field *var = field_var->var;
3101 struct hist_field *val = field_var->val;
3102
3103 var_val = val->fn(val, elt, rbe, rec);
3104 var_idx = var->var.idx;
3105
3106 if (val->flags & HIST_FIELD_FL_STRING) {
3107 char *str = elt_data->field_var_str[j++];
3108 char *val_str = (char *)(uintptr_t)var_val;
3109
ad452870 3110 strscpy(str, val_str, STR_VAR_LEN_MAX);
02205a67
TZ		 3111 var_val = (u64)(uintptr_t)str;
3112 }
3113 tracing_map_set_var(elt, var_idx, var_val);
3114 }
3115}
3116
3117static void update_field_vars(struct hist_trigger_data *hist_data,
3118 struct tracing_map_elt *elt,
3119 struct ring_buffer_event *rbe,
3120 void *rec)
3121{
3122 __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3123 hist_data->n_field_vars, 0);
3124}
3125
50450603
TZ		 3126static void update_max_vars(struct hist_trigger_data *hist_data,
3127 struct tracing_map_elt *elt,
3128 struct ring_buffer_event *rbe,
3129 void *rec)
3130{
3131 __update_field_vars(elt, rbe, rec, hist_data->max_vars,
3132 hist_data->n_max_vars, hist_data->n_field_var_str);
3133}
3134
02205a67
TZ		 3135static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3136 struct trace_event_file *file,
3137 char *name, int size, const char *type)
3138{
3139 struct hist_field *var;
3140 int idx;
3141
3142 if (find_var(hist_data, file, name) && !hist_data->remove) {
3143 var = ERR_PTR(-EINVAL);
3144 goto out;
3145 }
3146
3147 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3148 if (!var) {
3149 var = ERR_PTR(-ENOMEM);
3150 goto out;
3151 }
3152
3153 idx = tracing_map_add_var(hist_data->map);
3154 if (idx < 0) {
3155 kfree(var);
3156 var = ERR_PTR(-EINVAL);
3157 goto out;
3158 }
3159
3160 var->flags = HIST_FIELD_FL_VAR;
3161 var->var.idx = idx;
3162 var->var.hist_data = var->hist_data = hist_data;
3163 var->size = size;
3164 var->var.name = kstrdup(name, GFP_KERNEL);
3165 var->type = kstrdup(type, GFP_KERNEL);
3166 if (!var->var.name || !var->type) {
3167 kfree(var->var.name);
3168 kfree(var->type);
3169 kfree(var);
3170 var = ERR_PTR(-ENOMEM);
3171 }
3172 out:
3173 return var;
3174}
3175
3176static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3177 struct trace_event_file *file,
3178 char *field_name)
3179{
3180 struct hist_field *val = NULL, *var = NULL;
3181 unsigned long flags = HIST_FIELD_FL_VAR;
3182 struct field_var *field_var;
3183 int ret = 0;
3184
3185 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
f404da6e 3186 hist_err("Too many field variables defined: ", field_name);
02205a67
TZ		 3187 ret = -EINVAL;
3188 goto err;
3189 }
3190
3191 val = parse_atom(hist_data, file, field_name, &flags, NULL);
3192 if (IS_ERR(val)) {
f404da6e 3193 hist_err("Couldn't parse field variable: ", field_name);
02205a67
TZ		 3194 ret = PTR_ERR(val);
3195 goto err;
3196 }
3197
3198 var = create_var(hist_data, file, field_name, val->size, val->type);
3199 if (IS_ERR(var)) {
f404da6e 3200 hist_err("Couldn't create or find variable: ", field_name);
02205a67
TZ		 3201 kfree(val);
3202 ret = PTR_ERR(var);
3203 goto err;
3204 }
3205
3206 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3207 if (!field_var) {
3208 kfree(val);
3209 kfree(var);
3210 ret = -ENOMEM;
3211 goto err;
3212 }
3213
3214 field_var->var = var;
3215 field_var->val = val;
3216 out:
3217 return field_var;
3218 err:
3219 field_var = ERR_PTR(ret);
3220 goto out;
3221}
3222
3223/**
3224 * create_target_field_var - Automatically create a variable for a field
3225 * @target_hist_data: The target hist trigger
3226 * @subsys_name: Optional subsystem name
3227 * @event_name: Optional event name
3228 * @var_name: The name of the field (and the resulting variable)
3229 *
3230 * Hist trigger actions fetch data from variables, not directly from
3231 * events. However, for convenience, users are allowed to directly
3232 * specify an event field in an action, which will be automatically
3233 * converted into a variable on their behalf.
3234
3235 * This function creates a field variable with the name var_name on
3236 * the hist trigger currently being defined on the target event. If
3237 * subsys_name and event_name are specified, this function simply
3238 * verifies that they do in fact match the target event subsystem and
3239 * event name.
3240 *
3241 * Return: The variable created for the field.
3242 */
c282a386 3243static struct field_var *
02205a67
TZ		 3244create_target_field_var(struct hist_trigger_data *target_hist_data,
3245 char *subsys_name, char *event_name, char *var_name)
3246{
3247 struct trace_event_file *file = target_hist_data->event_file;
3248
3249 if (subsys_name) {
3250 struct trace_event_call *call;
3251
3252 if (!event_name)
3253 return NULL;
3254
3255 call = file->event_call;
3256
3257 if (strcmp(subsys_name, call->class->system) != 0)
3258 return NULL;
3259
3260 if (strcmp(event_name, trace_event_name(call)) != 0)
3261 return NULL;
3262 }
3263
3264 return create_field_var(target_hist_data, file, var_name);
3265}
3266
50450603
TZ		 3267static void onmax_print(struct seq_file *m,
3268 struct hist_trigger_data *hist_data,
3269 struct tracing_map_elt *elt,
3270 struct action_data *data)
3271{
3272 unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx;
3273
3274 seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx));
3275
3276 for (i = 0; i < hist_data->n_max_vars; i++) {
3277 struct hist_field *save_val = hist_data->max_vars[i]->val;
3278 struct hist_field *save_var = hist_data->max_vars[i]->var;
3279 u64 val;
3280
3281 save_var_idx = save_var->var.idx;
3282
3283 val = tracing_map_read_var(elt, save_var_idx);
3284
3285 if (save_val->flags & HIST_FIELD_FL_STRING) {
3286 seq_printf(m, " %s: %-32s", save_var->var.name,
3287 (char *)(uintptr_t)(val));
3288 } else
3289 seq_printf(m, " %s: %10llu", save_var->var.name, val);
3290 }
3291}
3292
3293static void onmax_save(struct hist_trigger_data *hist_data,
3294 struct tracing_map_elt *elt, void *rec,
3295 struct ring_buffer_event *rbe,
3296 struct action_data *data, u64 *var_ref_vals)
3297{
3298 unsigned int max_idx = data->onmax.max_var->var.idx;
3299 unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx;
3300
3301 u64 var_val, max_val;
3302
3303 var_val = var_ref_vals[max_var_ref_idx];
3304 max_val = tracing_map_read_var(elt, max_idx);
3305
3306 if (var_val <= max_val)
3307 return;
3308
3309 tracing_map_set_var(elt, max_idx, var_val);
3310
3311 update_max_vars(hist_data, elt, rbe, rec);
3312}
3313
3314static void onmax_destroy(struct action_data *data)
3315{
3316 unsigned int i;
3317
3318 destroy_hist_field(data->onmax.max_var, 0);
3319 destroy_hist_field(data->onmax.var, 0);
3320
3321 kfree(data->onmax.var_str);
3322 kfree(data->onmax.fn_name);
3323
3324 for (i = 0; i < data->n_params; i++)
3325 kfree(data->params[i]);
3326
3327 kfree(data);
3328}
3329
3330static int onmax_create(struct hist_trigger_data *hist_data,
3331 struct action_data *data)
3332{
3333 struct trace_event_file *file = hist_data->event_file;
3334 struct hist_field *var_field, *ref_field, *max_var;
3335 unsigned int var_ref_idx = hist_data->n_var_refs;
3336 struct field_var *field_var;
3337 char *onmax_var_str, *param;
3338 unsigned long flags;
3339 unsigned int i;
3340 int ret = 0;
3341
3342 onmax_var_str = data->onmax.var_str;
f404da6e
TZ		 3343 if (onmax_var_str[0] != '$') {
3344 hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str);
50450603 3345 return -EINVAL;
f404da6e 3346 }
50450603
TZ		 3347 onmax_var_str++;
3348
3349 var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
f404da6e
TZ		 3350 if (!var_field) {
3351 hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str);
50450603 3352 return -EINVAL;
f404da6e 3353 }
50450603
TZ		 3354
3355 flags = HIST_FIELD_FL_VAR_REF;
3356 ref_field = create_hist_field(hist_data, NULL, flags, NULL);
3357 if (!ref_field)
3358 return -ENOMEM;
3359
3360 if (init_var_ref(ref_field, var_field, NULL, NULL)) {
3361 destroy_hist_field(ref_field, 0);
3362 ret = -ENOMEM;
3363 goto out;
3364 }
3365 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3366 ref_field->var_ref_idx = hist_data->n_var_refs++;
3367 data->onmax.var = ref_field;
3368
3369 data->fn = onmax_save;
3370 data->onmax.max_var_ref_idx = var_ref_idx;
3371 max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
3372 if (IS_ERR(max_var)) {
f404da6e 3373 hist_err("onmax: Couldn't create onmax variable: ", "max");
50450603
TZ		 3374 ret = PTR_ERR(max_var);
3375 goto out;
3376 }
3377 data->onmax.max_var = max_var;
3378
3379 for (i = 0; i < data->n_params; i++) {
3380 param = kstrdup(data->params[i], GFP_KERNEL);
3381 if (!param) {
3382 ret = -ENOMEM;
3383 goto out;
3384 }
3385
3386 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3387 if (IS_ERR(field_var)) {
f404da6e 3388 hist_err("onmax: Couldn't create field variable: ", param);
50450603
TZ		 3389 ret = PTR_ERR(field_var);
3390 kfree(param);
3391 goto out;
3392 }
3393
3394 hist_data->max_vars[hist_data->n_max_vars++] = field_var;
3395 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3396 hist_data->n_max_var_str++;
3397
3398 kfree(param);
3399 }
3400 out:
3401 return ret;
3402}
3403
3404static int parse_action_params(char *params, struct action_data *data)
3405{
3406 char *param, *saved_param;
3407 int ret = 0;
3408
3409 while (params) {
3410 if (data->n_params >= SYNTH_FIELDS_MAX)
3411 goto out;
3412
3413 param = strsep(&params, ",");
3414 if (!param) {
3415 ret = -EINVAL;
3416 goto out;
3417 }
3418
3419 param = strstrip(param);
3420 if (strlen(param) < 2) {
f404da6e 3421 hist_err("Invalid action param: ", param);
50450603
TZ		 3422 ret = -EINVAL;
3423 goto out;
3424 }
3425
3426 saved_param = kstrdup(param, GFP_KERNEL);
3427 if (!saved_param) {
3428 ret = -ENOMEM;
3429 goto out;
3430 }
3431
3432 data->params[data->n_params++] = saved_param;
3433 }
3434 out:
3435 return ret;
3436}
3437
3438static struct action_data *onmax_parse(char *str)
3439{
3440 char *onmax_fn_name, *onmax_var_str;
3441 struct action_data *data;
3442 int ret = -EINVAL;
3443
3444 data = kzalloc(sizeof(*data), GFP_KERNEL);
3445 if (!data)
3446 return ERR_PTR(-ENOMEM);
3447
3448 onmax_var_str = strsep(&str, ")");
3449 if (!onmax_var_str || !str) {
3450 ret = -EINVAL;
3451 goto free;
3452 }
3453
3454 data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL);
3455 if (!data->onmax.var_str) {
3456 ret = -ENOMEM;
3457 goto free;
3458 }
3459
3460 strsep(&str, ".");
3461 if (!str)
3462 goto free;
3463
3464 onmax_fn_name = strsep(&str, "(");
3465 if (!onmax_fn_name || !str)
3466 goto free;
3467
3468 if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) {
3469 char *params = strsep(&str, ")");
3470
3471 if (!params) {
3472 ret = -EINVAL;
3473 goto free;
3474 }
3475
3476 ret = parse_action_params(params, data);
3477 if (ret)
3478 goto free;
3479 } else
3480 goto free;
3481
3482 data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL);
3483 if (!data->onmax.fn_name) {
3484 ret = -ENOMEM;
3485 goto free;
3486 }
3487 out:
3488 return data;
3489 free:
3490 onmax_destroy(data);
3491 data = ERR_PTR(ret);
3492 goto out;
3493}
3494
c282a386
TZ		 3495static void onmatch_destroy(struct action_data *data)
3496{
3497 unsigned int i;
3498
3499 mutex_lock(&synth_event_mutex);
3500
3501 kfree(data->onmatch.match_event);
3502 kfree(data->onmatch.match_event_system);
3503 kfree(data->onmatch.synth_event_name);
3504
3505 for (i = 0; i < data->n_params; i++)
3506 kfree(data->params[i]);
3507
3508 if (data->onmatch.synth_event)
3509 data->onmatch.synth_event->ref--;
3510
3511 kfree(data);
3512
3513 mutex_unlock(&synth_event_mutex);
3514}
3515
02205a67
TZ		 3516static void destroy_field_var(struct field_var *field_var)
3517{
3518 if (!field_var)
3519 return;
3520
3521 destroy_hist_field(field_var->var, 0);
3522 destroy_hist_field(field_var->val, 0);
3523
3524 kfree(field_var);
3525}
3526
3527static void destroy_field_vars(struct hist_trigger_data *hist_data)
3528{
3529 unsigned int i;
3530
3531 for (i = 0; i < hist_data->n_field_vars; i++)
3532 destroy_field_var(hist_data->field_vars[i]);
3533}
3534
c282a386
TZ		 3535static void save_field_var(struct hist_trigger_data *hist_data,
3536 struct field_var *field_var)
02205a67
TZ		 3537{
3538 hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3539
3540 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3541 hist_data->n_field_var_str++;
3542}
3543
c282a386
TZ		 3544
3545static void destroy_synth_var_refs(struct hist_trigger_data *hist_data)
3546{
3547 unsigned int i;
3548
3549 for (i = 0; i < hist_data->n_synth_var_refs; i++)
3550 destroy_hist_field(hist_data->synth_var_refs[i], 0);
3551}
3552
3553static void save_synth_var_ref(struct hist_trigger_data *hist_data,
3554 struct hist_field *var_ref)
3555{
3556 hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref;
3557
3558 hist_data->var_refs[hist_data->n_var_refs] = var_ref;
3559 var_ref->var_ref_idx = hist_data->n_var_refs++;
3560}
3561
3562static int check_synth_field(struct synth_event *event,
3563 struct hist_field *hist_field,
3564 unsigned int field_pos)
3565{
3566 struct synth_field *field;
3567
3568 if (field_pos >= event->n_fields)
3569 return -EINVAL;
3570
3571 field = event->fields[field_pos];
3572
3573 if (strcmp(field->type, hist_field->type) != 0)
3574 return -EINVAL;
3575
3576 return 0;
3577}
3578
c282a386
TZ		 3579static struct hist_field *
3580onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data,
3581 char *system, char *event, char *var)
3582{
3583 struct hist_field *hist_field;
3584
3585 var++; /* skip '$' */
3586
3587 hist_field = find_target_event_var(hist_data, system, event, var);
3588 if (!hist_field) {
3589 if (!system) {
3590 system = data->onmatch.match_event_system;
3591 event = data->onmatch.match_event;
3592 }
3593
3594 hist_field = find_event_var(hist_data, system, event, var);
3595 }
3596
f404da6e
TZ		 3597 if (!hist_field)
3598 hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var);
3599
c282a386
TZ		 3600 return hist_field;
3601}
3602
3603static struct hist_field *
3604onmatch_create_field_var(struct hist_trigger_data *hist_data,
3605 struct action_data *data, char *system,
3606 char *event, char *var)
3607{
3608 struct hist_field *hist_field = NULL;
3609 struct field_var *field_var;
3610
3611 /*
3612 * First try to create a field var on the target event (the
3613 * currently being defined). This will create a variable for
3614 * unqualified fields on the target event, or if qualified,
3615 * target fields that have qualified names matching the target.
3616 */
3617 field_var = create_target_field_var(hist_data, system, event, var);
3618
3619 if (field_var && !IS_ERR(field_var)) {
3620 save_field_var(hist_data, field_var);
3621 hist_field = field_var->var;
3622 } else {
3623 field_var = NULL;
3624 /*
3625 * If no explicit system.event is specfied, default to
3626 * looking for fields on the onmatch(system.event.xxx)
3627 * event.
3628 */
3629 if (!system) {
3630 system = data->onmatch.match_event_system;
3631 event = data->onmatch.match_event;
3632 }
3633
3634 /*
3635 * At this point, we're looking at a field on another
3636 * event. Because we can't modify a hist trigger on
3637 * another event to add a variable for a field, we need
3638 * to create a new trigger on that event and create the
3639 * variable at the same time.
3640 */
3641 hist_field = create_field_var_hist(hist_data, system, event, var);
3642 if (IS_ERR(hist_field))
3643 goto free;
3644 }
3645 out:
3646 return hist_field;
3647 free:
3648 destroy_field_var(field_var);
3649 hist_field = NULL;
3650 goto out;
3651}
3652
3653static int onmatch_create(struct hist_trigger_data *hist_data,
3654 struct trace_event_file *file,
3655 struct action_data *data)
3656{
3657 char *event_name, *param, *system = NULL;
3658 struct hist_field *hist_field, *var_ref;
3659 unsigned int i, var_ref_idx;
3660 unsigned int field_pos = 0;
3661 struct synth_event *event;
3662 int ret = 0;
3663
3664 mutex_lock(&synth_event_mutex);
3665 event = find_synth_event(data->onmatch.synth_event_name);
3666 if (!event) {
f404da6e 3667 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name);
c282a386
TZ		 3668 mutex_unlock(&synth_event_mutex);
3669 return -EINVAL;
3670 }
3671 event->ref++;
3672 mutex_unlock(&synth_event_mutex);
3673
3674 var_ref_idx = hist_data->n_var_refs;
3675
3676 for (i = 0; i < data->n_params; i++) {
3677 char *p;
3678
3679 p = param = kstrdup(data->params[i], GFP_KERNEL);
3680 if (!param) {
3681 ret = -ENOMEM;
3682 goto err;
3683 }
3684
3685 system = strsep(&param, ".");
3686 if (!param) {
3687 param = (char *)system;
3688 system = event_name = NULL;
3689 } else {
3690 event_name = strsep(&param, ".");
3691 if (!param) {
3692 kfree(p);
3693 ret = -EINVAL;
3694 goto err;
3695 }
3696 }
3697
3698 if (param[0] == '$')
3699 hist_field = onmatch_find_var(hist_data, data, system,
3700 event_name, param);
3701 else
3702 hist_field = onmatch_create_field_var(hist_data, data,
3703 system,
3704 event_name,
3705 param);
3706
3707 if (!hist_field) {
3708 kfree(p);
3709 ret = -EINVAL;
3710 goto err;
3711 }
3712
3713 if (check_synth_field(event, hist_field, field_pos) == 0) {
3714 var_ref = create_var_ref(hist_field, system, event_name);
3715 if (!var_ref) {
3716 kfree(p);
3717 ret = -ENOMEM;
3718 goto err;
3719 }
3720
3721 save_synth_var_ref(hist_data, var_ref);
3722 field_pos++;
3723 kfree(p);
3724 continue;
3725 }
3726
f404da6e
TZ		 3727 hist_err_event("onmatch: Param type doesn't match synthetic event field type: ",
3728 system, event_name, param);
c282a386
TZ		 3729 kfree(p);
3730 ret = -EINVAL;
3731 goto err;
3732 }
3733
3734 if (field_pos != event->n_fields) {
f404da6e 3735 hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name);
c282a386
TZ		 3736 ret = -EINVAL;
3737 goto err;
3738 }
3739
3740 data->fn = action_trace;
3741 data->onmatch.synth_event = event;
3742 data->onmatch.var_ref_idx = var_ref_idx;
3743 out:
3744 return ret;
3745 err:
3746 mutex_lock(&synth_event_mutex);
3747 event->ref--;
3748 mutex_unlock(&synth_event_mutex);
3749
3750 goto out;
3751}
3752
3753static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3754{
3755 char *match_event, *match_event_system;
3756 char *synth_event_name, *params;
3757 struct action_data *data;
3758 int ret = -EINVAL;
3759
3760 data = kzalloc(sizeof(*data), GFP_KERNEL);
3761 if (!data)
3762 return ERR_PTR(-ENOMEM);
3763
3764 match_event = strsep(&str, ")");
f404da6e
TZ		 3765 if (!match_event || !str) {
3766 hist_err("onmatch: Missing closing paren: ", match_event);
c282a386 3767 goto free;
f404da6e 3768 }
c282a386
TZ		 3769
3770 match_event_system = strsep(&match_event, ".");
f404da6e
TZ		 3771 if (!match_event) {
3772 hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
c282a386 3773 goto free;
f404da6e 3774 }
c282a386 3775
f404da6e
TZ		 3776 if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3777 hist_err_event("onmatch: Invalid subsystem or event name: ",
3778 match_event_system, match_event, NULL);
c282a386 3779 goto free;
f404da6e 3780 }
c282a386
TZ		 3781
3782 data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
3783 if (!data->onmatch.match_event) {
3784 ret = -ENOMEM;
3785 goto free;
3786 }
3787
3788 data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL);
3789 if (!data->onmatch.match_event_system) {
3790 ret = -ENOMEM;
3791 goto free;
3792 }
3793
3794 strsep(&str, ".");
f404da6e
TZ		 3795 if (!str) {
3796 hist_err("onmatch: Missing . after onmatch(): ", str);
c282a386 3797 goto free;
f404da6e 3798 }
c282a386
TZ		 3799
3800 synth_event_name = strsep(&str, "(");
f404da6e
TZ		 3801 if (!synth_event_name || !str) {
3802 hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name);
c282a386 3803 goto free;
f404da6e 3804 }
c282a386
TZ		 3805
3806 data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
3807 if (!data->onmatch.synth_event_name) {
3808 ret = -ENOMEM;
3809 goto free;
3810 }
3811
3812 params = strsep(&str, ")");
f404da6e
TZ		 3813 if (!params || !str || (str && strlen(str))) {
3814 hist_err("onmatch: Missing closing paramlist paren: ", params);
c282a386 3815 goto free;
f404da6e 3816 }
c282a386
TZ		 3817
3818 ret = parse_action_params(params, data);
3819 if (ret)
3820 goto free;
3821 out:
3822 return data;
3823 free:
3824 onmatch_destroy(data);
3825 data = ERR_PTR(ret);
3826 goto out;
3827}
3828
7ef224d1
TZ		 3829static int create_hitcount_val(struct hist_trigger_data *hist_data)
3830{
3831 hist_data->fields[HITCOUNT_IDX] =
30350d65 3832 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
7ef224d1
TZ		 3833 if (!hist_data->fields[HITCOUNT_IDX])
3834 return -ENOMEM;
3835
3836 hist_data->n_vals++;
30350d65 3837 hist_data->n_fields++;
7ef224d1
TZ		 3838
3839 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3840 return -EINVAL;
3841
3842 return 0;
3843}
3844
30350d65
TZ		 3845static int __create_val_field(struct hist_trigger_data *hist_data,
3846 unsigned int val_idx,
3847 struct trace_event_file *file,
3848 char *var_name, char *field_str,
3849 unsigned long flags)
f2606835 3850{
100719dc 3851 struct hist_field *hist_field;
f2606835
TZ		 3852 int ret = 0;
3853
100719dc
TZ		 3854 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3855 if (IS_ERR(hist_field)) {
3856 ret = PTR_ERR(hist_field);
f2606835
TZ		 3857 goto out;
3858 }
3859
100719dc
TZ		 3860 hist_data->fields[val_idx] = hist_field;
3861
f2606835 3862 ++hist_data->n_vals;
30350d65 3863 ++hist_data->n_fields;
f2606835 3864
30350d65 3865 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
f2606835
TZ		 3866 ret = -EINVAL;
3867 out:
3868 return ret;
3869}
3870
30350d65
TZ		 3871static int create_val_field(struct hist_trigger_data *hist_data,
3872 unsigned int val_idx,
3873 struct trace_event_file *file,
3874 char *field_str)
3875{
3876 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3877 return -EINVAL;
3878
3879 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3880}
3881
3882static int create_var_field(struct hist_trigger_data *hist_data,
3883 unsigned int val_idx,
3884 struct trace_event_file *file,
3885 char *var_name, char *expr_str)
3886{
3887 unsigned long flags = 0;
3888
3889 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3890 return -EINVAL;
f404da6e 3891
30350d65 3892 if (find_var(hist_data, file, var_name) && !hist_data->remove) {
f404da6e 3893 hist_err("Variable already defined: ", var_name);
30350d65
TZ		 3894 return -EINVAL;
3895 }
3896
3897 flags |= HIST_FIELD_FL_VAR;
3898 hist_data->n_vars++;
3899 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3900 return -EINVAL;
3901
3902 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3903}
3904
7ef224d1
TZ		 3905static int create_val_fields(struct hist_trigger_data *hist_data,
3906 struct trace_event_file *file)
3907{
f2606835 3908 char *fields_str, *field_str;
30350d65 3909 unsigned int i, j = 1;
7ef224d1
TZ		 3910 int ret;
3911
3912 ret = create_hitcount_val(hist_data);
f2606835
TZ		 3913 if (ret)
3914 goto out;
7ef224d1 3915
f2606835
TZ		 3916 fields_str = hist_data->attrs->vals_str;
3917 if (!fields_str)
3918 goto out;
3919
3920 strsep(&fields_str, "=");
3921 if (!fields_str)
3922 goto out;
3923
3924 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3925 j < TRACING_MAP_VALS_MAX; i++) {
3926 field_str = strsep(&fields_str, ",");
3927 if (!field_str)
3928 break;
30350d65 3929
f2606835
TZ		 3930 if (strcmp(field_str, "hitcount") == 0)
3931 continue;
30350d65 3932
f2606835
TZ		 3933 ret = create_val_field(hist_data, j++, file, field_str);
3934 if (ret)
3935 goto out;
3936 }
30350d65 3937
f2606835
TZ		 3938 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3939 ret = -EINVAL;
3940 out:
7ef224d1
TZ		 3941 return ret;
3942}
3943
3944static int create_key_field(struct hist_trigger_data *hist_data,
3945 unsigned int key_idx,
76a3b0c8 3946 unsigned int key_offset,
7ef224d1
TZ		 3947 struct trace_event_file *file,
3948 char *field_str)
3949{
30350d65 3950 struct hist_field *hist_field = NULL;
100719dc 3951
7ef224d1
TZ		 3952 unsigned long flags = 0;
3953 unsigned int key_size;
3954 int ret = 0;
3955
30350d65 3956 if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
7ef224d1
TZ		 3957 return -EINVAL;
3958
3959 flags |= HIST_FIELD_FL_KEY;
3960
69a0200c
TZ		 3961 if (strcmp(field_str, "stacktrace") == 0) {
3962 flags |= HIST_FIELD_FL_STACKTRACE;
3963 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
30350d65 3964 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
69a0200c 3965 } else {
100719dc
TZ		 3966 hist_field = parse_expr(hist_data, file, field_str, flags,
3967 NULL, 0);
3968 if (IS_ERR(hist_field)) {
3969 ret = PTR_ERR(hist_field);
3970 goto out;
69a0200c
TZ		 3971 }
3972
067fe038 3973 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
f404da6e 3974 hist_err("Using variable references as keys not supported: ", field_str);
067fe038
TZ		 3975 destroy_hist_field(hist_field, 0);
3976 ret = -EINVAL;
3977 goto out;
3978 }
3979
100719dc 3980 key_size = hist_field->size;
7ef224d1
TZ		 3981 }
3982
100719dc 3983 hist_data->fields[key_idx] = hist_field;
7ef224d1
TZ		 3984
3985 key_size = ALIGN(key_size, sizeof(u64));
3986 hist_data->fields[key_idx]->size = key_size;
76a3b0c8 3987 hist_data->fields[key_idx]->offset = key_offset;
100719dc 3988
76a3b0c8 3989 hist_data->key_size += key_size;
100719dc 3990
7ef224d1
TZ		 3991 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3992 ret = -EINVAL;
3993 goto out;
3994 }
3995
3996 hist_data->n_keys++;
30350d65 3997 hist_data->n_fields++;
7ef224d1
TZ		 3998
3999 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4000 return -EINVAL;
4001
4002 ret = key_size;
4003 out:
4004 return ret;
4005}
4006
4007static int create_key_fields(struct hist_trigger_data *hist_data,
4008 struct trace_event_file *file)
4009{
76a3b0c8 4010 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
7ef224d1
TZ		 4011 char *fields_str, *field_str;
4012 int ret = -EINVAL;
4013
4014 fields_str = hist_data->attrs->keys_str;
4015 if (!fields_str)
4016 goto out;
4017
4018 strsep(&fields_str, "=");
4019 if (!fields_str)
4020 goto out;
4021
76a3b0c8 4022 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
7ef224d1
TZ		 4023 field_str = strsep(&fields_str, ",");
4024 if (!field_str)
4025 break;
76a3b0c8
TZ		 4026 ret = create_key_field(hist_data, i, key_offset,
4027 file, field_str);
7ef224d1
TZ		 4028 if (ret < 0)
4029 goto out;
76a3b0c8 4030 key_offset += ret;
7ef224d1
TZ		 4031 }
4032 if (fields_str) {
4033 ret = -EINVAL;
4034 goto out;
4035 }
4036 ret = 0;
4037 out:
4038 return ret;
4039}
4040
30350d65
TZ		 4041static int create_var_fields(struct hist_trigger_data *hist_data,
4042 struct trace_event_file *file)
4043{
4044 unsigned int i, j = hist_data->n_vals;
4045 int ret = 0;
4046
4047 unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4048
4049 for (i = 0; i < n_vars; i++) {
4050 char *var_name = hist_data->attrs->var_defs.name[i];
4051 char *expr = hist_data->attrs->var_defs.expr[i];
4052
4053 ret = create_var_field(hist_data, j++, file, var_name, expr);
4054 if (ret)
4055 goto out;
4056 }
4057 out:
4058 return ret;
4059}
4060
4061static void free_var_defs(struct hist_trigger_data *hist_data)
4062{
4063 unsigned int i;
4064
4065 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4066 kfree(hist_data->attrs->var_defs.name[i]);
4067 kfree(hist_data->attrs->var_defs.expr[i]);
4068 }
4069
4070 hist_data->attrs->var_defs.n_vars = 0;
4071}
4072
4073static int parse_var_defs(struct hist_trigger_data *hist_data)
4074{
4075 char *s, *str, *var_name, *field_str;
4076 unsigned int i, j, n_vars = 0;
4077 int ret = 0;
4078
4079 for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4080 str = hist_data->attrs->assignment_str[i];
4081 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4082 field_str = strsep(&str, ",");
4083 if (!field_str)
4084 break;
4085
4086 var_name = strsep(&field_str, "=");
4087 if (!var_name || !field_str) {
f404da6e 4088 hist_err("Malformed assignment: ", var_name);
30350d65
TZ		 4089 ret = -EINVAL;
4090 goto free;
4091 }
4092
4093 if (n_vars == TRACING_MAP_VARS_MAX) {
f404da6e 4094 hist_err("Too many variables defined: ", var_name);
30350d65
TZ		 4095 ret = -EINVAL;
4096 goto free;
4097 }
4098
4099 s = kstrdup(var_name, GFP_KERNEL);
4100 if (!s) {
4101 ret = -ENOMEM;
4102 goto free;
4103 }
4104 hist_data->attrs->var_defs.name[n_vars] = s;
4105
4106 s = kstrdup(field_str, GFP_KERNEL);
4107 if (!s) {
4108 kfree(hist_data->attrs->var_defs.name[n_vars]);
4109 ret = -ENOMEM;
4110 goto free;
4111 }
4112 hist_data->attrs->var_defs.expr[n_vars++] = s;
4113
4114 hist_data->attrs->var_defs.n_vars = n_vars;
4115 }
4116 }
4117
4118 return ret;
4119 free:
4120 free_var_defs(hist_data);
4121
4122 return ret;
4123}
4124
7ef224d1
TZ		 4125static int create_hist_fields(struct hist_trigger_data *hist_data,
4126 struct trace_event_file *file)
4127{
4128 int ret;
4129
30350d65
TZ		 4130 ret = parse_var_defs(hist_data);
4131 if (ret)
4132 goto out;
4133
7ef224d1
TZ		 4134 ret = create_val_fields(hist_data, file);
4135 if (ret)
4136 goto out;
4137
30350d65 4138 ret = create_var_fields(hist_data, file);
7ef224d1
TZ		 4139 if (ret)
4140 goto out;
4141
30350d65
TZ		 4142 ret = create_key_fields(hist_data, file);
4143 if (ret)
4144 goto out;
7ef224d1 4145 out:
30350d65
TZ		 4146 free_var_defs(hist_data);
4147
7ef224d1
TZ		 4148 return ret;
4149}
4150
e62347d2
TZ		 4151static int is_descending(const char *str)
4152{
4153 if (!str)
4154 return 0;
4155
4156 if (strcmp(str, "descending") == 0)
4157 return 1;
4158
4159 if (strcmp(str, "ascending") == 0)
4160 return 0;
4161
4162 return -EINVAL;
4163}
4164
7ef224d1
TZ		 4165static int create_sort_keys(struct hist_trigger_data *hist_data)
4166{
e62347d2 4167 char *fields_str = hist_data->attrs->sort_key_str;
e62347d2
TZ		 4168 struct tracing_map_sort_key *sort_key;
4169 int descending, ret = 0;
30350d65 4170 unsigned int i, j, k;
e62347d2
TZ		 4171
4172 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4173
4174 if (!fields_str)
4175 goto out;
4176
4177 strsep(&fields_str, "=");
4178 if (!fields_str) {
4179 ret = -EINVAL;
4180 goto out;
4181 }
4182
4183 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
85013256 4184 struct hist_field *hist_field;
e62347d2 4185 char *field_str, *field_name;
85013256 4186 const char *test_name;
e62347d2
TZ		 4187
4188 sort_key = &hist_data->sort_keys[i];
4189
4190 field_str = strsep(&fields_str, ",");
4191 if (!field_str) {
4192 if (i == 0)
4193 ret = -EINVAL;
4194 break;
4195 }
4196
4197 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4198 ret = -EINVAL;
4199 break;
4200 }
7ef224d1 4201
e62347d2
TZ		 4202 field_name = strsep(&field_str, ".");
4203 if (!field_name) {
4204 ret = -EINVAL;
4205 break;
4206 }
4207
4208 if (strcmp(field_name, "hitcount") == 0) {
4209 descending = is_descending(field_str);
4210 if (descending < 0) {
4211 ret = descending;
4212 break;
4213 }
4214 sort_key->descending = descending;
4215 continue;
4216 }
7ef224d1 4217
30350d65
TZ		 4218 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4219 unsigned int idx;
4220
85013256 4221 hist_field = hist_data->fields[j];
30350d65
TZ		 4222 if (hist_field->flags & HIST_FIELD_FL_VAR)
4223 continue;
4224
4225 idx = k++;
4226
85013256
TZ		 4227 test_name = hist_field_name(hist_field, 0);
4228
4229 if (strcmp(field_name, test_name) == 0) {
30350d65 4230 sort_key->field_idx = idx;
e62347d2
TZ		 4231 descending = is_descending(field_str);
4232 if (descending < 0) {
4233 ret = descending;
4234 goto out;
4235 }
4236 sort_key->descending = descending;
4237 break;
4238 }
4239 }
4240 if (j == hist_data->n_fields) {
4241 ret = -EINVAL;
4242 break;
4243 }
4244 }
30350d65 4245
e62347d2
TZ		 4246 hist_data->n_sort_keys = i;
4247 out:
7ef224d1
TZ		 4248 return ret;
4249}
4250
0212e2aa
TZ		 4251static void destroy_actions(struct hist_trigger_data *hist_data)
4252{
4253 unsigned int i;
4254
4255 for (i = 0; i < hist_data->n_actions; i++) {
4256 struct action_data *data = hist_data->actions[i];
4257
c282a386
TZ		 4258 if (data->fn == action_trace)
4259 onmatch_destroy(data);
50450603
TZ		 4260 else if (data->fn == onmax_save)
4261 onmax_destroy(data);
c282a386
TZ		 4262 else
4263 kfree(data);
0212e2aa
TZ		 4264 }
4265}
4266
4267static int parse_actions(struct hist_trigger_data *hist_data)
4268{
c282a386
TZ		 4269 struct trace_array *tr = hist_data->event_file->tr;
4270 struct action_data *data;
0212e2aa
TZ		 4271 unsigned int i;
4272 int ret = 0;
4273 char *str;
4274
4275 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4276 str = hist_data->attrs->action_str[i];
c282a386
TZ		 4277
4278 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) {
4279 char *action_str = str + strlen("onmatch(");
4280
4281 data = onmatch_parse(tr, action_str);
4282 if (IS_ERR(data)) {
4283 ret = PTR_ERR(data);
4284 break;
4285 }
4286 data->fn = action_trace;
50450603
TZ		 4287 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) {
4288 char *action_str = str + strlen("onmax(");
4289
4290 data = onmax_parse(action_str);
4291 if (IS_ERR(data)) {
4292 ret = PTR_ERR(data);
4293 break;
4294 }
4295 data->fn = onmax_save;
c282a386
TZ		 4296 } else {
4297 ret = -EINVAL;
4298 break;
4299 }
4300
4301 hist_data->actions[hist_data->n_actions++] = data;
0212e2aa
TZ		 4302 }
4303
4304 return ret;
4305}
4306
4307static int create_actions(struct hist_trigger_data *hist_data,
4308 struct trace_event_file *file)
4309{
4310 struct action_data *data;
4311 unsigned int i;
4312 int ret = 0;
4313
4314 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4315 data = hist_data->actions[i];
c282a386
TZ		 4316
4317 if (data->fn == action_trace) {
4318 ret = onmatch_create(hist_data, file, data);
4319 if (ret)
4320 return ret;
50450603
TZ		 4321 } else if (data->fn == onmax_save) {
4322 ret = onmax_create(hist_data, data);
4323 if (ret)
4324 return ret;
c282a386 4325 }
0212e2aa
TZ		 4326 }
4327
4328 return ret;
4329}
4330
50450603
TZ		 4331static void print_actions(struct seq_file *m,
4332 struct hist_trigger_data *hist_data,
4333 struct tracing_map_elt *elt)
4334{
4335 unsigned int i;
4336
4337 for (i = 0; i < hist_data->n_actions; i++) {
4338 struct action_data *data = hist_data->actions[i];
4339
4340 if (data->fn == onmax_save)
4341 onmax_print(m, hist_data, elt, data);
4342 }
4343}
4344
4345static void print_onmax_spec(struct seq_file *m,
4346 struct hist_trigger_data *hist_data,
4347 struct action_data *data)
4348{
4349 unsigned int i;
4350
4351 seq_puts(m, ":onmax(");
4352 seq_printf(m, "%s", data->onmax.var_str);
4353 seq_printf(m, ").%s(", data->onmax.fn_name);
4354
4355 for (i = 0; i < hist_data->n_max_vars; i++) {
4356 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name);
4357 if (i < hist_data->n_max_vars - 1)
4358 seq_puts(m, ",");
4359 }
4360 seq_puts(m, ")");
4361}
4362
c282a386
TZ		 4363static void print_onmatch_spec(struct seq_file *m,
4364 struct hist_trigger_data *hist_data,
4365 struct action_data *data)
4366{
4367 unsigned int i;
4368
4369 seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system,
4370 data->onmatch.match_event);
4371
4372 seq_printf(m, "%s(", data->onmatch.synth_event->name);
4373
4374 for (i = 0; i < data->n_params; i++) {
4375 if (i)
4376 seq_puts(m, ",");
4377 seq_printf(m, "%s", data->params[i]);
4378 }
4379
4380 seq_puts(m, ")");
4381}
4382
48f79473
TZ		 4383static bool actions_match(struct hist_trigger_data *hist_data,
4384 struct hist_trigger_data *hist_data_test)
4385{
4386 unsigned int i, j;
4387
4388 if (hist_data->n_actions != hist_data_test->n_actions)
4389 return false;
4390
4391 for (i = 0; i < hist_data->n_actions; i++) {
4392 struct action_data *data = hist_data->actions[i];
4393 struct action_data *data_test = hist_data_test->actions[i];
4394
4395 if (data->fn != data_test->fn)
4396 return false;
4397
4398 if (data->n_params != data_test->n_params)
4399 return false;
4400
4401 for (j = 0; j < data->n_params; j++) {
4402 if (strcmp(data->params[j], data_test->params[j]) != 0)
4403 return false;
4404 }
4405
4406 if (data->fn == action_trace) {
4407 if (strcmp(data->onmatch.synth_event_name,
4408 data_test->onmatch.synth_event_name) != 0)
4409 return false;
4410 if (strcmp(data->onmatch.match_event_system,
4411 data_test->onmatch.match_event_system) != 0)
4412 return false;
4413 if (strcmp(data->onmatch.match_event,
4414 data_test->onmatch.match_event) != 0)
4415 return false;
4416 } else if (data->fn == onmax_save) {
4417 if (strcmp(data->onmax.var_str,
4418 data_test->onmax.var_str) != 0)
4419 return false;
4420 if (strcmp(data->onmax.fn_name,
4421 data_test->onmax.fn_name) != 0)
4422 return false;
4423 }
4424 }
4425
4426 return true;
4427}
4428
4429
c282a386
TZ		 4430static void print_actions_spec(struct seq_file *m,
4431 struct hist_trigger_data *hist_data)
4432{
4433 unsigned int i;
4434
4435 for (i = 0; i < hist_data->n_actions; i++) {
4436 struct action_data *data = hist_data->actions[i];
4437
4438 if (data->fn == action_trace)
4439 print_onmatch_spec(m, hist_data, data);
50450603
TZ		 4440 else if (data->fn == onmax_save)
4441 print_onmax_spec(m, hist_data, data);
c282a386
TZ		 4442 }
4443}
4444
02205a67
TZ		 4445static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4446{
4447 unsigned int i;
4448
4449 for (i = 0; i < hist_data->n_field_var_hists; i++) {
4450 kfree(hist_data->field_var_hists[i]->cmd);
4451 kfree(hist_data->field_var_hists[i]);
4452 }
4453}
4454
7ef224d1
TZ		 4455static void destroy_hist_data(struct hist_trigger_data *hist_data)
4456{
0212e2aa
TZ		 4457 if (!hist_data)
4458 return;
4459
7ef224d1
TZ		 4460 destroy_hist_trigger_attrs(hist_data->attrs);
4461 destroy_hist_fields(hist_data);
4462 tracing_map_destroy(hist_data->map);
0212e2aa
TZ		 4463
4464 destroy_actions(hist_data);
02205a67
TZ		 4465 destroy_field_vars(hist_data);
4466 destroy_field_var_hists(hist_data);
c282a386 4467 destroy_synth_var_refs(hist_data);
0212e2aa 4468
7ef224d1
TZ		 4469 kfree(hist_data);
4470}
4471
4472static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4473{
4474 struct tracing_map *map = hist_data->map;
4475 struct ftrace_event_field *field;
4476 struct hist_field *hist_field;
b28d7b2d 4477 int i, idx = 0;
7ef224d1
TZ		 4478
4479 for_each_hist_field(i, hist_data) {
4480 hist_field = hist_data->fields[i];
4481 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4482 tracing_map_cmp_fn_t cmp_fn;
4483
4484 field = hist_field->field;
4485
69a0200c
TZ		 4486 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4487 cmp_fn = tracing_map_cmp_none;
ad42febe
TZ		 4488 else if (!field)
4489 cmp_fn = tracing_map_cmp_num(hist_field->size,
4490 hist_field->is_signed);
69a0200c 4491 else if (is_string_field(field))
7ef224d1
TZ		 4492 cmp_fn = tracing_map_cmp_string;
4493 else
4494 cmp_fn = tracing_map_cmp_num(field->size,
4495 field->is_signed);
76a3b0c8
TZ		 4496 idx = tracing_map_add_key_field(map,
4497 hist_field->offset,
4498 cmp_fn);
30350d65 4499 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
7ef224d1
TZ		 4500 idx = tracing_map_add_sum_field(map);
4501
4502 if (idx < 0)
4503 return idx;
30350d65
TZ		 4504
4505 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4506 idx = tracing_map_add_var(map);
4507 if (idx < 0)
4508 return idx;
4509 hist_field->var.idx = idx;
4510 hist_field->var.hist_data = hist_data;
4511 }
7ef224d1
TZ		 4512 }
4513
4514 return 0;
4515}
4516
4517static struct hist_trigger_data *
4518create_hist_data(unsigned int map_bits,
4519 struct hist_trigger_attrs *attrs,
30350d65
TZ		 4520 struct trace_event_file *file,
4521 bool remove)
7ef224d1 4522{
6b4827ad 4523 const struct tracing_map_ops *map_ops = NULL;
7ef224d1
TZ		 4524 struct hist_trigger_data *hist_data;
4525 int ret = 0;
4526
4527 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4528 if (!hist_data)
4529 return ERR_PTR(-ENOMEM);
4530
4531 hist_data->attrs = attrs;
30350d65 4532 hist_data->remove = remove;
067fe038 4533 hist_data->event_file = file;
7ef224d1 4534
0212e2aa
TZ		 4535 ret = parse_actions(hist_data);
4536 if (ret)
4537 goto free;
4538
7ef224d1
TZ		 4539 ret = create_hist_fields(hist_data, file);
4540 if (ret)
4541 goto free;
4542
4543 ret = create_sort_keys(hist_data);
4544 if (ret)
4545 goto free;
4546
af6a29bc 4547 map_ops = &hist_trigger_elt_data_ops;
6b4827ad 4548
7ef224d1 4549 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6b4827ad 4550 map_ops, hist_data);
7ef224d1
TZ		 4551 if (IS_ERR(hist_data->map)) {
4552 ret = PTR_ERR(hist_data->map);
4553 hist_data->map = NULL;
4554 goto free;
4555 }
4556
4557 ret = create_tracing_map_fields(hist_data);
4558 if (ret)
4559 goto free;
7ef224d1
TZ		 4560 out:
4561 return hist_data;
4562 free:
4563 hist_data->attrs = NULL;
4564
4565 destroy_hist_data(hist_data);
4566
4567 hist_data = ERR_PTR(ret);
4568
4569 goto out;
4570}
4571
4572static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
fbd302cb 4573 struct tracing_map_elt *elt, void *rec,
067fe038
TZ		 4574 struct ring_buffer_event *rbe,
4575 u64 *var_ref_vals)
7ef224d1 4576{
067fe038 4577 struct hist_elt_data *elt_data;
7ef224d1 4578 struct hist_field *hist_field;
30350d65 4579 unsigned int i, var_idx;
7ef224d1
TZ		 4580 u64 hist_val;
4581
067fe038
TZ		 4582 elt_data = elt->private_data;
4583 elt_data->var_ref_vals = var_ref_vals;
4584
7ef224d1
TZ		 4585 for_each_hist_val_field(i, hist_data) {
4586 hist_field = hist_data->fields[i];
df35d93b 4587 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 4588 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4589 var_idx = hist_field->var.idx;
4590 tracing_map_set_var(elt, var_idx, hist_val);
4591 continue;
4592 }
7ef224d1
TZ		 4593 tracing_map_update_sum(elt, i, hist_val);
4594 }
30350d65
TZ		 4595
4596 for_each_hist_key_field(i, hist_data) {
4597 hist_field = hist_data->fields[i];
4598 if (hist_field->flags & HIST_FIELD_FL_VAR) {
df35d93b 4599 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 4600 var_idx = hist_field->var.idx;
4601 tracing_map_set_var(elt, var_idx, hist_val);
4602 }
4603 }
02205a67
TZ		 4604
4605 update_field_vars(hist_data, elt, rbe, rec);
7ef224d1
TZ		 4606}
4607
6a475cb1
TZ		 4608static inline void add_to_key(char *compound_key, void *key,
4609 struct hist_field *key_field, void *rec)
4610{
4611 size_t size = key_field->size;
4612
4613 if (key_field->flags & HIST_FIELD_FL_STRING) {
4614 struct ftrace_event_field *field;
4615
4616 field = key_field->field;
4617 if (field->filter_type == FILTER_DYN_STRING)
4618 size = *(u32 *)(rec + field->offset) >> 16;
4619 else if (field->filter_type == FILTER_PTR_STRING)
4620 size = strlen(key);
4621 else if (field->filter_type == FILTER_STATIC_STRING)
4622 size = field->size;
4623
4624 /* ensure NULL-termination */
4625 if (size > key_field->size - 1)
4626 size = key_field->size - 1;
6a475cb1 4627
ebca08d7
TZ		 4628 strncpy(compound_key + key_field->offset, (char *)key, size);
4629 } else
4630 memcpy(compound_key + key_field->offset, key, size);
6a475cb1
TZ		 4631}
4632
0212e2aa
TZ		 4633static void
4634hist_trigger_actions(struct hist_trigger_data *hist_data,
4635 struct tracing_map_elt *elt, void *rec,
4636 struct ring_buffer_event *rbe, u64 *var_ref_vals)
4637{
4638 struct action_data *data;
4639 unsigned int i;
4640
4641 for (i = 0; i < hist_data->n_actions; i++) {
4642 data = hist_data->actions[i];
4643 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals);
4644 }
4645}
4646
1ac4f51c 4647static void event_hist_trigger(struct event_trigger_data *data, void *rec,
fbd302cb 4648 struct ring_buffer_event *rbe)
7ef224d1
TZ		 4649{
4650 struct hist_trigger_data *hist_data = data->private_data;
6a475cb1 4651 bool use_compound_key = (hist_data->n_keys > 1);
69a0200c 4652 unsigned long entries[HIST_STACKTRACE_DEPTH];
067fe038 4653 u64 var_ref_vals[TRACING_MAP_VARS_MAX];
76a3b0c8 4654 char compound_key[HIST_KEY_SIZE_MAX];
df35d93b 4655 struct tracing_map_elt *elt = NULL;
69a0200c 4656 struct stack_trace stacktrace;
7ef224d1 4657 struct hist_field *key_field;
7ef224d1
TZ		 4658 u64 field_contents;
4659 void *key = NULL;
4660 unsigned int i;
4661
6a475cb1 4662 memset(compound_key, 0, hist_data->key_size);
76a3b0c8 4663
7ef224d1
TZ		 4664 for_each_hist_key_field(i, hist_data) {
4665 key_field = hist_data->fields[i];
4666
69a0200c
TZ		 4667 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4668 stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
4669 stacktrace.entries = entries;
4670 stacktrace.nr_entries = 0;
4671 stacktrace.skip = HIST_STACKTRACE_SKIP;
76a3b0c8 4672
69a0200c
TZ		 4673 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
4674 save_stack_trace(&stacktrace);
4675
4676 key = entries;
4677 } else {
df35d93b 4678 field_contents = key_field->fn(key_field, elt, rbe, rec);
6a475cb1 4679 if (key_field->flags & HIST_FIELD_FL_STRING) {
69a0200c 4680 key = (void *)(unsigned long)field_contents;
6a475cb1
TZ		 4681 use_compound_key = true;
4682 } else
69a0200c 4683 key = (void *)&field_contents;
76a3b0c8 4684 }
6a475cb1
TZ		 4685
4686 if (use_compound_key)
4687 add_to_key(compound_key, key, key_field, rec);
7ef224d1
TZ		 4688 }
4689
6a475cb1 4690 if (use_compound_key)
76a3b0c8
TZ		 4691 key = compound_key;
4692
067fe038
TZ		 4693 if (hist_data->n_var_refs &&
4694 !resolve_var_refs(hist_data, key, var_ref_vals, false))
4695 return;
4696
7ef224d1 4697 elt = tracing_map_insert(hist_data->map, key);
067fe038
TZ		 4698 if (!elt)
4699 return;
4700
4701 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
0212e2aa
TZ		 4702
4703 if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4704 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals);
7ef224d1
TZ		 4705}
4706
69a0200c
TZ		 4707static void hist_trigger_stacktrace_print(struct seq_file *m,
4708 unsigned long *stacktrace_entries,
4709 unsigned int max_entries)
4710{
4711 char str[KSYM_SYMBOL_LEN];
4712 unsigned int spaces = 8;
4713 unsigned int i;
4714
4715 for (i = 0; i < max_entries; i++) {
4716 if (stacktrace_entries[i] == ULONG_MAX)
4717 return;
4718
4719 seq_printf(m, "%*c", 1 + spaces, ' ');
4720 sprint_symbol(str, stacktrace_entries[i]);
4721 seq_printf(m, "%s\n", str);
4722 }
4723}
4724
7ef224d1
TZ		 4725static void
4726hist_trigger_entry_print(struct seq_file *m,
4727 struct hist_trigger_data *hist_data, void *key,
4728 struct tracing_map_elt *elt)
4729{
4730 struct hist_field *key_field;
c6afad49 4731 char str[KSYM_SYMBOL_LEN];
69a0200c 4732 bool multiline = false;
85013256 4733 const char *field_name;
7ef224d1
TZ		 4734 unsigned int i;
4735 u64 uval;
4736
4737 seq_puts(m, "{ ");
4738
4739 for_each_hist_key_field(i, hist_data) {
4740 key_field = hist_data->fields[i];
4741
4742 if (i > hist_data->n_vals)
4743 seq_puts(m, ", ");
4744
85013256
TZ		 4745 field_name = hist_field_name(key_field, 0);
4746
0c4a6b46
TZ		 4747 if (key_field->flags & HIST_FIELD_FL_HEX) {
4748 uval = *(u64 *)(key + key_field->offset);
85013256 4749 seq_printf(m, "%s: %llx", field_name, uval);
c6afad49
TZ		 4750 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4751 uval = *(u64 *)(key + key_field->offset);
4752 sprint_symbol_no_offset(str, uval);
85013256
TZ		 4753 seq_printf(m, "%s: [%llx] %-45s", field_name,
4754 uval, str);
c6afad49
TZ		 4755 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4756 uval = *(u64 *)(key + key_field->offset);
4757 sprint_symbol(str, uval);
85013256
TZ		 4758 seq_printf(m, "%s: [%llx] %-55s", field_name,
4759 uval, str);
6b4827ad 4760 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 4761 struct hist_elt_data *elt_data = elt->private_data;
4762 char *comm;
4763
4764 if (WARN_ON_ONCE(!elt_data))
4765 return;
4766
4767 comm = elt_data->comm;
6b4827ad
TZ		 4768
4769 uval = *(u64 *)(key + key_field->offset);
85013256
TZ		 4770 seq_printf(m, "%s: %-16s[%10llu]", field_name,
4771 comm, uval);
31696198
TZ		 4772 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4773 const char *syscall_name;
4774
4775 uval = *(u64 *)(key + key_field->offset);
4776 syscall_name = get_syscall_name(uval);
4777 if (!syscall_name)
4778 syscall_name = "unknown_syscall";
4779
85013256
TZ		 4780 seq_printf(m, "%s: %-30s[%3llu]", field_name,
4781 syscall_name, uval);
69a0200c
TZ		 4782 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4783 seq_puts(m, "stacktrace:\n");
4784 hist_trigger_stacktrace_print(m,
4785 key + key_field->offset,
4786 HIST_STACKTRACE_DEPTH);
4787 multiline = true;
4b94f5b7 4788 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
85013256 4789 seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4b94f5b7 4790 *(u64 *)(key + key_field->offset));
0c4a6b46 4791 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
85013256 4792 seq_printf(m, "%s: %-50s", field_name,
76a3b0c8 4793 (char *)(key + key_field->offset));
7ef224d1 4794 } else {
76a3b0c8 4795 uval = *(u64 *)(key + key_field->offset);
85013256 4796 seq_printf(m, "%s: %10llu", field_name, uval);
7ef224d1
TZ		 4797 }
4798 }
4799
69a0200c
TZ		 4800 if (!multiline)
4801 seq_puts(m, " ");
4802
4803 seq_puts(m, "}");
7ef224d1
TZ		 4804
4805 seq_printf(m, " hitcount: %10llu",
4806 tracing_map_read_sum(elt, HITCOUNT_IDX));
4807
f2606835 4808 for (i = 1; i < hist_data->n_vals; i++) {
85013256
TZ		 4809 field_name = hist_field_name(hist_data->fields[i], 0);
4810
100719dc
TZ		 4811 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4812 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
30350d65
TZ		 4813 continue;
4814
0c4a6b46 4815 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
85013256 4816 seq_printf(m, " %s: %10llx", field_name,
0c4a6b46
TZ		 4817 tracing_map_read_sum(elt, i));
4818 } else {
85013256 4819 seq_printf(m, " %s: %10llu", field_name,
0c4a6b46
TZ		 4820 tracing_map_read_sum(elt, i));
4821 }
f2606835
TZ		 4822 }
4823
50450603
TZ		 4824 print_actions(m, hist_data, elt);
4825
7ef224d1
TZ		 4826 seq_puts(m, "\n");
4827}
4828
4829static int print_entries(struct seq_file *m,
4830 struct hist_trigger_data *hist_data)
4831{
4832 struct tracing_map_sort_entry **sort_entries = NULL;
4833 struct tracing_map *map = hist_data->map;
d50c744e 4834 int i, n_entries;
7ef224d1
TZ		 4835
4836 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4837 hist_data->n_sort_keys,
4838 &sort_entries);
4839 if (n_entries < 0)
4840 return n_entries;
4841
4842 for (i = 0; i < n_entries; i++)
4843 hist_trigger_entry_print(m, hist_data,
4844 sort_entries[i]->key,
4845 sort_entries[i]->elt);
4846
4847 tracing_map_destroy_sort_entries(sort_entries, n_entries);
4848
4849 return n_entries;
4850}
4851
52a7f16d
TZ		 4852static void hist_trigger_show(struct seq_file *m,
4853 struct event_trigger_data *data, int n)
7ef224d1 4854{
7ef224d1 4855 struct hist_trigger_data *hist_data;
6e7a2398 4856 int n_entries;
7ef224d1 4857
52a7f16d
TZ		 4858 if (n > 0)
4859 seq_puts(m, "\n\n");
7ef224d1
TZ		 4860
4861 seq_puts(m, "# event histogram\n#\n# trigger info: ");
4862 data->ops->print(m, data->ops, data);
52a7f16d 4863 seq_puts(m, "#\n\n");
7ef224d1
TZ		 4864
4865 hist_data = data->private_data;
4866 n_entries = print_entries(m, hist_data);
6e7a2398 4867 if (n_entries < 0)
7ef224d1 4868 n_entries = 0;
7ef224d1
TZ		 4869
4870 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
4871 (u64)atomic64_read(&hist_data->map->hits),
4872 n_entries, (u64)atomic64_read(&hist_data->map->drops));
52a7f16d
TZ		 4873}
4874
4875static int hist_show(struct seq_file *m, void *v)
4876{
4877 struct event_trigger_data *data;
4878 struct trace_event_file *event_file;
4879 int n = 0, ret = 0;
4880
4881 mutex_lock(&event_mutex);
4882
4883 event_file = event_file_data(m->private);
4884 if (unlikely(!event_file)) {
4885 ret = -ENODEV;
4886 goto out_unlock;
4887 }
4888
4889 list_for_each_entry_rcu(data, &event_file->triggers, list) {
4890 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4891 hist_trigger_show(m, data, n++);
4892 }
4893
f404da6e
TZ		 4894 if (have_hist_err()) {
4895 seq_printf(m, "\nERROR: %s\n", hist_err_str);
4896 seq_printf(m, " Last command: %s\n", last_hist_cmd);
4897 }
4898
7ef224d1
TZ		 4899 out_unlock:
4900 mutex_unlock(&event_mutex);
4901
4902 return ret;
4903}
4904
4905static int event_hist_open(struct inode *inode, struct file *file)
4906{
4907 return single_open(file, hist_show, file);
4908}
4909
4910const struct file_operations event_hist_fops = {
4911 .open = event_hist_open,
4912 .read = seq_read,
4913 .llseek = seq_lseek,
4914 .release = single_release,
4915};
4916
4917static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
4918{
85013256
TZ		 4919 const char *field_name = hist_field_name(hist_field, 0);
4920
30350d65
TZ		 4921 if (hist_field->var.name)
4922 seq_printf(m, "%s=", hist_field->var.name);
4923
0ae7961e 4924 if (hist_field->flags & HIST_FIELD_FL_CPU)
8b7622bf 4925 seq_puts(m, "cpu");
067fe038 4926 else if (field_name) {
7e8b88a3
TZ		 4927 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
4928 hist_field->flags & HIST_FIELD_FL_ALIAS)
067fe038 4929 seq_putc(m, '$');
ad42febe 4930 seq_printf(m, "%s", field_name);
0ae7961e
TZ		 4931 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
4932 seq_puts(m, "common_timestamp");
608940da
TZ		 4933
4934 if (hist_field->flags) {
4935 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
4936 !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
4937 const char *flags = get_hist_field_flags(hist_field);
4938
4939 if (flags)
4940 seq_printf(m, ".%s", flags);
4941 }
4942 }
7ef224d1
TZ		 4943}
4944
4945static int event_hist_trigger_print(struct seq_file *m,
4946 struct event_trigger_ops *ops,
4947 struct event_trigger_data *data)
4948{
4949 struct hist_trigger_data *hist_data = data->private_data;
30350d65
TZ		 4950 struct hist_field *field;
4951 bool have_var = false;
7ef224d1
TZ		 4952 unsigned int i;
4953
5463bfda
TZ		 4954 seq_puts(m, "hist:");
4955
4956 if (data->name)
4957 seq_printf(m, "%s:", data->name);
4958
4959 seq_puts(m, "keys=");
7ef224d1
TZ		 4960
4961 for_each_hist_key_field(i, hist_data) {
30350d65 4962 field = hist_data->fields[i];
7ef224d1
TZ		 4963
4964 if (i > hist_data->n_vals)
4965 seq_puts(m, ",");
4966
30350d65 4967 if (field->flags & HIST_FIELD_FL_STACKTRACE)
69a0200c
TZ		 4968 seq_puts(m, "stacktrace");
4969 else
30350d65 4970 hist_field_print(m, field);
7ef224d1
TZ		 4971 }
4972
4973 seq_puts(m, ":vals=");
f2606835
TZ		 4974
4975 for_each_hist_val_field(i, hist_data) {
30350d65
TZ		 4976 field = hist_data->fields[i];
4977 if (field->flags & HIST_FIELD_FL_VAR) {
4978 have_var = true;
4979 continue;
4980 }
4981
f2606835
TZ		 4982 if (i == HITCOUNT_IDX)
4983 seq_puts(m, "hitcount");
4984 else {
4985 seq_puts(m, ",");
30350d65
TZ		 4986 hist_field_print(m, field);
4987 }
4988 }
4989
4990 if (have_var) {
4991 unsigned int n = 0;
4992
4993 seq_puts(m, ":");
4994
4995 for_each_hist_val_field(i, hist_data) {
4996 field = hist_data->fields[i];
4997
4998 if (field->flags & HIST_FIELD_FL_VAR) {
4999 if (n++)
5000 seq_puts(m, ",");
5001 hist_field_print(m, field);
5002 }
f2606835
TZ		 5003 }
5004 }
7ef224d1
TZ		 5005
5006 seq_puts(m, ":sort=");
e62347d2
TZ		 5007
5008 for (i = 0; i < hist_data->n_sort_keys; i++) {
5009 struct tracing_map_sort_key *sort_key;
30350d65
TZ		 5010 unsigned int idx, first_key_idx;
5011
5012 /* skip VAR vals */
5013 first_key_idx = hist_data->n_vals - hist_data->n_vars;
e62347d2
TZ		 5014
5015 sort_key = &hist_data->sort_keys[i];
ad42febe
TZ		 5016 idx = sort_key->field_idx;
5017
1a361dfc 5018 if (WARN_ON(idx >= HIST_FIELDS_MAX))
ad42febe 5019 return -EINVAL;
e62347d2
TZ		 5020
5021 if (i > 0)
5022 seq_puts(m, ",");
5023
ad42febe 5024 if (idx == HITCOUNT_IDX)
e62347d2 5025 seq_puts(m, "hitcount");
30350d65
TZ		 5026 else {
5027 if (idx >= first_key_idx)
5028 idx += hist_data->n_vars;
e62347d2 5029 hist_field_print(m, hist_data->fields[idx]);
30350d65 5030 }
e62347d2
TZ		 5031
5032 if (sort_key->descending)
5033 seq_puts(m, ".descending");
5034 }
7ef224d1 5035 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
a4072fe8
TZ		 5036 if (hist_data->enable_timestamps)
5037 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
7ef224d1 5038
c282a386
TZ		 5039 print_actions_spec(m, hist_data);
5040
7ef224d1
TZ		 5041 if (data->filter_str)
5042 seq_printf(m, " if %s", data->filter_str);
5043
83e99914
TZ		 5044 if (data->paused)
5045 seq_puts(m, " [paused]");
5046 else
5047 seq_puts(m, " [active]");
7ef224d1
TZ		 5048
5049 seq_putc(m, '\n');
5050
5051 return 0;
5052}
5053
5463bfda
TZ		 5054static int event_hist_trigger_init(struct event_trigger_ops *ops,
5055 struct event_trigger_data *data)
5056{
5057 struct hist_trigger_data *hist_data = data->private_data;
5058
5059 if (!data->ref && hist_data->attrs->name)
5060 save_named_trigger(hist_data->attrs->name, data);
5061
5062 data->ref++;
5063
5064 return 0;
5065}
5066
02205a67
TZ		 5067static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5068{
5069 struct trace_event_file *file;
5070 unsigned int i;
5071 char *cmd;
5072 int ret;
5073
5074 for (i = 0; i < hist_data->n_field_var_hists; i++) {
5075 file = hist_data->field_var_hists[i]->hist_data->event_file;
5076 cmd = hist_data->field_var_hists[i]->cmd;
5077 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5078 "!hist", "hist", cmd);
5079 }
5080}
5081
7ef224d1
TZ		 5082static void event_hist_trigger_free(struct event_trigger_ops *ops,
5083 struct event_trigger_data *data)
5084{
5085 struct hist_trigger_data *hist_data = data->private_data;
5086
5087 if (WARN_ON_ONCE(data->ref <= 0))
5088 return;
5089
5090 data->ref--;
5091 if (!data->ref) {
5463bfda
TZ		 5092 if (data->name)
5093 del_named_trigger(data);
067fe038 5094
7ef224d1 5095 trigger_data_free(data);
067fe038
TZ		 5096
5097 remove_hist_vars(hist_data);
5098
02205a67
TZ		 5099 unregister_field_var_hists(hist_data);
5100
7ef224d1
TZ		 5101 destroy_hist_data(hist_data);
5102 }
5103}
5104
5105static struct event_trigger_ops event_hist_trigger_ops = {
5106 .func = event_hist_trigger,
5107 .print = event_hist_trigger_print,
5463bfda 5108 .init = event_hist_trigger_init,
7ef224d1
TZ		 5109 .free = event_hist_trigger_free,
5110};
5111
5463bfda
TZ		 5112static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5113 struct event_trigger_data *data)
5114{
5115 data->ref++;
5116
5117 save_named_trigger(data->named_data->name, data);
5118
5119 event_hist_trigger_init(ops, data->named_data);
5120
5121 return 0;
5122}
5123
5124static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5125 struct event_trigger_data *data)
5126{
5127 if (WARN_ON_ONCE(data->ref <= 0))
5128 return;
5129
5130 event_hist_trigger_free(ops, data->named_data);
5131
5132 data->ref--;
5133 if (!data->ref) {
5134 del_named_trigger(data);
5135 trigger_data_free(data);
5136 }
5137}
5138
5139static struct event_trigger_ops event_hist_trigger_named_ops = {
5140 .func = event_hist_trigger,
5141 .print = event_hist_trigger_print,
5142 .init = event_hist_trigger_named_init,
5143 .free = event_hist_trigger_named_free,
5144};
5145
7ef224d1
TZ		 5146static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5147 char *param)
5148{
5149 return &event_hist_trigger_ops;
5150}
5151
e86ae9ba
TZ		 5152static void hist_clear(struct event_trigger_data *data)
5153{
5154 struct hist_trigger_data *hist_data = data->private_data;
e86ae9ba 5155
5463bfda
TZ		 5156 if (data->name)
5157 pause_named_trigger(data);
e86ae9ba 5158
e0a568dc 5159 tracepoint_synchronize_unregister();
e86ae9ba
TZ		 5160
5161 tracing_map_clear(hist_data->map);
5162
5463bfda
TZ		 5163 if (data->name)
5164 unpause_named_trigger(data);
5165}
5166
5167static bool compatible_field(struct ftrace_event_field *field,
5168 struct ftrace_event_field *test_field)
5169{
5170 if (field == test_field)
5171 return true;
5172 if (field == NULL || test_field == NULL)
5173 return false;
5174 if (strcmp(field->name, test_field->name) != 0)
5175 return false;
5176 if (strcmp(field->type, test_field->type) != 0)
5177 return false;
5178 if (field->size != test_field->size)
5179 return false;
5180 if (field->is_signed != test_field->is_signed)
5181 return false;
5182
5183 return true;
e86ae9ba
TZ		 5184}
5185
52a7f16d 5186static bool hist_trigger_match(struct event_trigger_data *data,
5463bfda
TZ		 5187 struct event_trigger_data *data_test,
5188 struct event_trigger_data *named_data,
5189 bool ignore_filter)
52a7f16d
TZ		 5190{
5191 struct tracing_map_sort_key *sort_key, *sort_key_test;
5192 struct hist_trigger_data *hist_data, *hist_data_test;
5193 struct hist_field *key_field, *key_field_test;
5194 unsigned int i;
5195
5463bfda
TZ		 5196 if (named_data && (named_data != data_test) &&
5197 (named_data != data_test->named_data))
5198 return false;
5199
5200 if (!named_data && is_named_trigger(data_test))
5201 return false;
5202
52a7f16d
TZ		 5203 hist_data = data->private_data;
5204 hist_data_test = data_test->private_data;
5205
5206 if (hist_data->n_vals != hist_data_test->n_vals ||
5207 hist_data->n_fields != hist_data_test->n_fields ||
5208 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5209 return false;
5210
5463bfda
TZ		 5211 if (!ignore_filter) {
5212 if ((data->filter_str && !data_test->filter_str) ||
5213 (!data->filter_str && data_test->filter_str))
5214 return false;
5215 }
52a7f16d
TZ		 5216
5217 for_each_hist_field(i, hist_data) {
5218 key_field = hist_data->fields[i];
5219 key_field_test = hist_data_test->fields[i];
5220
5221 if (key_field->flags != key_field_test->flags)
5222 return false;
5463bfda 5223 if (!compatible_field(key_field->field, key_field_test->field))
52a7f16d
TZ		 5224 return false;
5225 if (key_field->offset != key_field_test->offset)
5226 return false;
ad42febe
TZ		 5227 if (key_field->size != key_field_test->size)
5228 return false;
5229 if (key_field->is_signed != key_field_test->is_signed)
5230 return false;
1a361dfc
TZ		 5231 if (!!key_field->var.name != !!key_field_test->var.name)
5232 return false;
5233 if (key_field->var.name &&
5234 strcmp(key_field->var.name, key_field_test->var.name) != 0)
5235 return false;
52a7f16d
TZ		 5236 }
5237
5238 for (i = 0; i < hist_data->n_sort_keys; i++) {
5239 sort_key = &hist_data->sort_keys[i];
5240 sort_key_test = &hist_data_test->sort_keys[i];
5241
5242 if (sort_key->field_idx != sort_key_test->field_idx ||
5243 sort_key->descending != sort_key_test->descending)
5244 return false;
5245 }
5246
5463bfda 5247 if (!ignore_filter && data->filter_str &&
52a7f16d
TZ		 5248 (strcmp(data->filter_str, data_test->filter_str) != 0))
5249 return false;
5250
48f79473
TZ		 5251 if (!actions_match(hist_data, hist_data_test))
5252 return false;
5253
52a7f16d
TZ		 5254 return true;
5255}
5256
7ef224d1
TZ		 5257static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5258 struct event_trigger_data *data,
5259 struct trace_event_file *file)
5260{
83e99914 5261 struct hist_trigger_data *hist_data = data->private_data;
5463bfda 5262 struct event_trigger_data *test, *named_data = NULL;
7ef224d1
TZ		 5263 int ret = 0;
5264
5463bfda
TZ		 5265 if (hist_data->attrs->name) {
5266 named_data = find_named_trigger(hist_data->attrs->name);
5267 if (named_data) {
5268 if (!hist_trigger_match(data, named_data, named_data,
5269 true)) {
f404da6e 5270 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
5463bfda
TZ		 5271 ret = -EINVAL;
5272 goto out;
5273 }
5274 }
5275 }
5276
5277 if (hist_data->attrs->name && !named_data)
5278 goto new;
5279
7ef224d1
TZ		 5280 list_for_each_entry_rcu(test, &file->triggers, list) {
5281 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 5282 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d 5283 continue;
83e99914
TZ		 5284 if (hist_data->attrs->pause)
5285 test->paused = true;
5286 else if (hist_data->attrs->cont)
5287 test->paused = false;
e86ae9ba
TZ		 5288 else if (hist_data->attrs->clear)
5289 hist_clear(test);
f404da6e
TZ		 5290 else {
5291 hist_err("Hist trigger already exists", NULL);
83e99914 5292 ret = -EEXIST;
f404da6e 5293 }
7ef224d1
TZ		 5294 goto out;
5295 }
5296 }
5463bfda 5297 new:
e86ae9ba 5298 if (hist_data->attrs->cont || hist_data->attrs->clear) {
f404da6e 5299 hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
83e99914
TZ		 5300 ret = -ENOENT;
5301 goto out;
5302 }
5303
7522c03a
TZ		 5304 if (hist_data->attrs->pause)
5305 data->paused = true;
5306
5463bfda 5307 if (named_data) {
5463bfda
TZ		 5308 data->private_data = named_data->private_data;
5309 set_named_trigger_data(data, named_data);
5310 data->ops = &event_hist_trigger_named_ops;
5311 }
5312
7ef224d1
TZ		 5313 if (data->ops->init) {
5314 ret = data->ops->init(data->ops, data);
5315 if (ret < 0)
5316 goto out;
5317 }
5318
a4072fe8
TZ		 5319 if (hist_data->enable_timestamps) {
5320 char *clock = hist_data->attrs->clock;
5321
5322 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5323 if (ret) {
5324 hist_err("Couldn't set trace_clock: ", clock);
5325 goto out;
5326 }
7ef224d1 5327
ad42febe 5328 tracing_set_time_stamp_abs(file->tr, true);
a4072fe8
TZ		 5329 }
5330
5331 if (named_data)
5332 destroy_hist_data(hist_data);
5333
5334 ret++;
067fe038
TZ		 5335 out:
5336 return ret;
5337}
5338
5339static int hist_trigger_enable(struct event_trigger_data *data,
5340 struct trace_event_file *file)
5341{
5342 int ret = 0;
5343
5344 list_add_tail_rcu(&data->list, &file->triggers);
5345
5346 update_cond_flag(file);
ad42febe 5347
7ef224d1
TZ		 5348 if (trace_event_trigger_enable_disable(file, 1) < 0) {
5349 list_del_rcu(&data->list);
5350 update_cond_flag(file);
5351 ret--;
5352 }
067fe038 5353
7ef224d1
TZ		 5354 return ret;
5355}
5356
4b147936
TZ		 5357static bool have_hist_trigger_match(struct event_trigger_data *data,
5358 struct trace_event_file *file)
5359{
5360 struct hist_trigger_data *hist_data = data->private_data;
5361 struct event_trigger_data *test, *named_data = NULL;
5362 bool match = false;
5363
5364 if (hist_data->attrs->name)
5365 named_data = find_named_trigger(hist_data->attrs->name);
5366
5367 list_for_each_entry_rcu(test, &file->triggers, list) {
5368 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5369 if (hist_trigger_match(data, test, named_data, false)) {
5370 match = true;
5371 break;
5372 }
5373 }
5374 }
5375
5376 return match;
5377}
5378
067fe038
TZ		 5379static bool hist_trigger_check_refs(struct event_trigger_data *data,
5380 struct trace_event_file *file)
5381{
5382 struct hist_trigger_data *hist_data = data->private_data;
5383 struct event_trigger_data *test, *named_data = NULL;
5384
5385 if (hist_data->attrs->name)
5386 named_data = find_named_trigger(hist_data->attrs->name);
5387
5388 list_for_each_entry_rcu(test, &file->triggers, list) {
5389 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5390 if (!hist_trigger_match(data, test, named_data, false))
5391 continue;
5392 hist_data = test->private_data;
5393 if (check_var_refs(hist_data))
5394 return true;
5395 break;
5396 }
5397 }
5398
5399 return false;
5400}
5401
52a7f16d
TZ		 5402static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5403 struct event_trigger_data *data,
5404 struct trace_event_file *file)
5405{
5463bfda
TZ		 5406 struct hist_trigger_data *hist_data = data->private_data;
5407 struct event_trigger_data *test, *named_data = NULL;
52a7f16d
TZ		 5408 bool unregistered = false;
5409
5463bfda
TZ		 5410 if (hist_data->attrs->name)
5411 named_data = find_named_trigger(hist_data->attrs->name);
5412
52a7f16d
TZ		 5413 list_for_each_entry_rcu(test, &file->triggers, list) {
5414 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 5415 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d
TZ		 5416 continue;
5417 unregistered = true;
5418 list_del_rcu(&test->list);
5419 trace_event_trigger_enable_disable(file, 0);
5420 update_cond_flag(file);
5421 break;
5422 }
5423 }
5424
5425 if (unregistered && test->ops->free)
5426 test->ops->free(test->ops, test);
ad42febe
TZ		 5427
5428 if (hist_data->enable_timestamps) {
30350d65 5429 if (!hist_data->remove || unregistered)
ad42febe
TZ		 5430 tracing_set_time_stamp_abs(file->tr, false);
5431 }
52a7f16d
TZ		 5432}
5433
067fe038
TZ		 5434static bool hist_file_check_refs(struct trace_event_file *file)
5435{
5436 struct hist_trigger_data *hist_data;
5437 struct event_trigger_data *test;
5438
5439 list_for_each_entry_rcu(test, &file->triggers, list) {
5440 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5441 hist_data = test->private_data;
5442 if (check_var_refs(hist_data))
5443 return true;
5444 }
5445 }
5446
5447 return false;
5448}
5449
52a7f16d
TZ		 5450static void hist_unreg_all(struct trace_event_file *file)
5451{
47c18569 5452 struct event_trigger_data *test, *n;
ad42febe 5453 struct hist_trigger_data *hist_data;
4b147936
TZ		 5454 struct synth_event *se;
5455 const char *se_name;
52a7f16d 5456
067fe038
TZ		 5457 if (hist_file_check_refs(file))
5458 return;
5459
47c18569 5460 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d 5461 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
ad42febe 5462 hist_data = test->private_data;
52a7f16d
TZ		 5463 list_del_rcu(&test->list);
5464 trace_event_trigger_enable_disable(file, 0);
4b147936
TZ		 5465
5466 mutex_lock(&synth_event_mutex);
5467 se_name = trace_event_name(file->event_call);
5468 se = find_synth_event(se_name);
5469 if (se)
5470 se->ref--;
5471 mutex_unlock(&synth_event_mutex);
5472
52a7f16d 5473 update_cond_flag(file);
ad42febe
TZ		 5474 if (hist_data->enable_timestamps)
5475 tracing_set_time_stamp_abs(file->tr, false);
52a7f16d
TZ		 5476 if (test->ops->free)
5477 test->ops->free(test->ops, test);
5478 }
5479 }
5480}
5481
7ef224d1
TZ		 5482static int event_hist_trigger_func(struct event_command *cmd_ops,
5483 struct trace_event_file *file,
5484 char *glob, char *cmd, char *param)
5485{
5486 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5487 struct event_trigger_data *trigger_data;
5488 struct hist_trigger_attrs *attrs;
5489 struct event_trigger_ops *trigger_ops;
5490 struct hist_trigger_data *hist_data;
4b147936
TZ		 5491 struct synth_event *se;
5492 const char *se_name;
30350d65 5493 bool remove = false;
ec5ce098 5494 char *trigger, *p;
7ef224d1
TZ		 5495 int ret = 0;
5496
f404da6e
TZ		 5497 if (glob && strlen(glob)) {
5498 last_cmd_set(param);
5499 hist_err_clear();
5500 }
5501
7ef224d1
TZ		 5502 if (!param)
5503 return -EINVAL;
5504
30350d65
TZ		 5505 if (glob[0] == '!')
5506 remove = true;
5507
ec5ce098
TZ		 5508 /*
5509 * separate the trigger from the filter (k:v [if filter])
5510 * allowing for whitespace in the trigger
5511 */
5512 p = trigger = param;
5513 do {
5514 p = strstr(p, "if");
5515 if (!p)
5516 break;
5517 if (p == param)
5518 return -EINVAL;
5519 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5520 p++;
5521 continue;
5522 }
5523 if (p >= param + strlen(param) - strlen("if") - 1)
5524 return -EINVAL;
5525 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') {
5526 p++;
5527 continue;
5528 }
5529 break;
5530 } while (p);
5531
5532 if (!p)
5533 param = NULL;
5534 else {
5535 *(p - 1) = '\0';
5536 param = strstrip(p);
5537 trigger = strstrip(trigger);
5538 }
7ef224d1
TZ		 5539
5540 attrs = parse_hist_trigger_attrs(trigger);
5541 if (IS_ERR(attrs))
5542 return PTR_ERR(attrs);
5543
5544 if (attrs->map_bits)
5545 hist_trigger_bits = attrs->map_bits;
5546
30350d65 5547 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
7ef224d1
TZ		 5548 if (IS_ERR(hist_data)) {
5549 destroy_hist_trigger_attrs(attrs);
5550 return PTR_ERR(hist_data);
5551 }
5552
5553 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5554
7ef224d1 5555 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
4b147936
TZ		 5556 if (!trigger_data) {
5557 ret = -ENOMEM;
7ef224d1 5558 goto out_free;
4b147936 5559 }
7ef224d1
TZ		 5560
5561 trigger_data->count = -1;
5562 trigger_data->ops = trigger_ops;
5563 trigger_data->cmd_ops = cmd_ops;
5564
5565 INIT_LIST_HEAD(&trigger_data->list);
5566 RCU_INIT_POINTER(trigger_data->filter, NULL);
5567
5568 trigger_data->private_data = hist_data;
5569
52a7f16d
TZ		 5570 /* if param is non-empty, it's supposed to be a filter */
5571 if (param && cmd_ops->set_filter) {
5572 ret = cmd_ops->set_filter(param, trigger_data, file);
5573 if (ret < 0)
5574 goto out_free;
5575 }
5576
30350d65 5577 if (remove) {
4b147936
TZ		 5578 if (!have_hist_trigger_match(trigger_data, file))
5579 goto out_free;
5580
067fe038
TZ		 5581 if (hist_trigger_check_refs(trigger_data, file)) {
5582 ret = -EBUSY;
5583 goto out_free;
5584 }
5585
7ef224d1 5586 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
4b147936
TZ		 5587
5588 mutex_lock(&synth_event_mutex);
5589 se_name = trace_event_name(file->event_call);
5590 se = find_synth_event(se_name);
5591 if (se)
5592 se->ref--;
5593 mutex_unlock(&synth_event_mutex);
5594
7ef224d1
TZ		 5595 ret = 0;
5596 goto out_free;
5597 }
5598
7ef224d1
TZ		 5599 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5600 /*
5601 * The above returns on success the # of triggers registered,
5602 * but if it didn't register any it returns zero. Consider no
5603 * triggers registered a failure too.
5604 */
5605 if (!ret) {
e86ae9ba 5606 if (!(attrs->pause || attrs->cont || attrs->clear))
83e99914 5607 ret = -ENOENT;
7ef224d1
TZ		 5608 goto out_free;
5609 } else if (ret < 0)
5610 goto out_free;
067fe038
TZ		 5611
5612 if (get_named_trigger_data(trigger_data))
5613 goto enable;
5614
5615 if (has_hist_vars(hist_data))
5616 save_hist_vars(hist_data);
5617
0212e2aa
TZ		 5618 ret = create_actions(hist_data, file);
5619 if (ret)
5620 goto out_unreg;
5621
067fe038
TZ		 5622 ret = tracing_map_init(hist_data->map);
5623 if (ret)
5624 goto out_unreg;
5625enable:
5626 ret = hist_trigger_enable(trigger_data, file);
5627 if (ret)
5628 goto out_unreg;
5629
4b147936
TZ		 5630 mutex_lock(&synth_event_mutex);
5631 se_name = trace_event_name(file->event_call);
5632 se = find_synth_event(se_name);
5633 if (se)
5634 se->ref++;
5635 mutex_unlock(&synth_event_mutex);
5636
7ef224d1
TZ		 5637 /* Just return zero, not the number of registered triggers */
5638 ret = 0;
5639 out:
f404da6e
TZ		 5640 if (ret == 0)
5641 hist_err_clear();
5642
7ef224d1 5643 return ret;
067fe038
TZ		 5644 out_unreg:
5645 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7ef224d1
TZ		 5646 out_free:
5647 if (cmd_ops->set_filter)
5648 cmd_ops->set_filter(NULL, trigger_data, NULL);
5649
067fe038
TZ		 5650 remove_hist_vars(hist_data);
5651
7ef224d1
TZ		 5652 kfree(trigger_data);
5653
5654 destroy_hist_data(hist_data);
5655 goto out;
5656}
5657
5658static struct event_command trigger_hist_cmd = {
5659 .name = "hist",
5660 .trigger_type = ETT_EVENT_HIST,
5661 .flags = EVENT_CMD_FL_NEEDS_REC,
5662 .func = event_hist_trigger_func,
5663 .reg = hist_register_trigger,
52a7f16d
TZ		 5664 .unreg = hist_unregister_trigger,
5665 .unreg_all = hist_unreg_all,
7ef224d1
TZ		 5666 .get_trigger_ops = event_hist_get_trigger_ops,
5667 .set_filter = set_trigger_filter,
5668};
5669
5670__init int register_trigger_hist_cmd(void)
5671{
5672 int ret;
5673
5674 ret = register_event_command(&trigger_hist_cmd);
5675 WARN_ON(ret < 0);
5676
5677 return ret;
5678}
d0bad49b
TZ		 5679
5680static void
1ac4f51c
TZ		 5681hist_enable_trigger(struct event_trigger_data *data, void *rec,
5682 struct ring_buffer_event *event)
d0bad49b
TZ		 5683{
5684 struct enable_trigger_data *enable_data = data->private_data;
5685 struct event_trigger_data *test;
5686
5687 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
5688 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5689 if (enable_data->enable)
5690 test->paused = false;
5691 else
5692 test->paused = true;
d0bad49b
TZ		 5693 }
5694 }
5695}
5696
5697static void
1ac4f51c
TZ		 5698hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5699 struct ring_buffer_event *event)
d0bad49b
TZ		 5700{
5701 if (!data->count)
5702 return;
5703
5704 if (data->count != -1)
5705 (data->count)--;
5706
1ac4f51c 5707 hist_enable_trigger(data, rec, event);
d0bad49b
TZ		 5708}
5709
5710static struct event_trigger_ops hist_enable_trigger_ops = {
5711 .func = hist_enable_trigger,
5712 .print = event_enable_trigger_print,
5713 .init = event_trigger_init,
5714 .free = event_enable_trigger_free,
5715};
5716
5717static struct event_trigger_ops hist_enable_count_trigger_ops = {
5718 .func = hist_enable_count_trigger,
5719 .print = event_enable_trigger_print,
5720 .init = event_trigger_init,
5721 .free = event_enable_trigger_free,
5722};
5723
5724static struct event_trigger_ops hist_disable_trigger_ops = {
5725 .func = hist_enable_trigger,
5726 .print = event_enable_trigger_print,
5727 .init = event_trigger_init,
5728 .free = event_enable_trigger_free,
5729};
5730
5731static struct event_trigger_ops hist_disable_count_trigger_ops = {
5732 .func = hist_enable_count_trigger,
5733 .print = event_enable_trigger_print,
5734 .init = event_trigger_init,
5735 .free = event_enable_trigger_free,
5736};
5737
5738static struct event_trigger_ops *
5739hist_enable_get_trigger_ops(char *cmd, char *param)
5740{
5741 struct event_trigger_ops *ops;
5742 bool enable;
5743
5744 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5745
5746 if (enable)
5747 ops = param ? &hist_enable_count_trigger_ops :
5748 &hist_enable_trigger_ops;
5749 else
5750 ops = param ? &hist_disable_count_trigger_ops :
5751 &hist_disable_trigger_ops;
5752
5753 return ops;
5754}
5755
52a7f16d
TZ		 5756static void hist_enable_unreg_all(struct trace_event_file *file)
5757{
47c18569 5758 struct event_trigger_data *test, *n;
52a7f16d 5759
47c18569 5760 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d
TZ		 5761 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5762 list_del_rcu(&test->list);
5763 update_cond_flag(file);
5764 trace_event_trigger_enable_disable(file, 0);
5765 if (test->ops->free)
5766 test->ops->free(test->ops, test);
5767 }
5768 }
5769}
5770
d0bad49b
TZ		 5771static struct event_command trigger_hist_enable_cmd = {
5772 .name = ENABLE_HIST_STR,
5773 .trigger_type = ETT_HIST_ENABLE,
5774 .func = event_enable_trigger_func,
5775 .reg = event_enable_register_trigger,
5776 .unreg = event_enable_unregister_trigger,
52a7f16d 5777 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 5778 .get_trigger_ops = hist_enable_get_trigger_ops,
5779 .set_filter = set_trigger_filter,
5780};
5781
5782static struct event_command trigger_hist_disable_cmd = {
5783 .name = DISABLE_HIST_STR,
5784 .trigger_type = ETT_HIST_ENABLE,
5785 .func = event_enable_trigger_func,
5786 .reg = event_enable_register_trigger,
5787 .unreg = event_enable_unregister_trigger,
52a7f16d 5788 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 5789 .get_trigger_ops = hist_enable_get_trigger_ops,
5790 .set_filter = set_trigger_filter,
5791};
5792
5793static __init void unregister_trigger_hist_enable_disable_cmds(void)
5794{
5795 unregister_event_command(&trigger_hist_enable_cmd);
5796 unregister_event_command(&trigger_hist_disable_cmd);
5797}
5798
5799__init int register_trigger_hist_enable_disable_cmds(void)
5800{
5801 int ret;
5802
5803 ret = register_event_command(&trigger_hist_enable_cmd);
5804 if (WARN_ON(ret < 0))
5805 return ret;
5806 ret = register_event_command(&trigger_hist_disable_cmd);
5807 if (WARN_ON(ret < 0))
5808 unregister_trigger_hist_enable_disable_cmds();
5809
5810 return ret;
5811}
4b147936
TZ		 5812
5813static __init int trace_events_hist_init(void)
5814{
5815 struct dentry *entry = NULL;
5816 struct dentry *d_tracer;
5817 int err = 0;
5818
5819 d_tracer = tracing_init_dentry();
5820 if (IS_ERR(d_tracer)) {
5821 err = PTR_ERR(d_tracer);
5822 goto err;
5823 }
5824
5825 entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
5826 NULL, &synth_events_fops);
5827 if (!entry) {
5828 err = -ENODEV;
5829 goto err;
5830 }
5831
5832 return err;
5833 err:
5834 pr_warn("Could not create tracefs 'synthetic_events' entry\n");
5835
5836 return err;
5837}
5838
5839fs_initcall(trace_events_hist_init);
Mirror https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git