]> git.ipfire.org Git - thirdparty/kernel/stable.git/blame - kernel/trace/trace_events_hist.c
projects / thirdparty / kernel / stable.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tracing: Add 'onmax' hist trigger action support
[thirdparty/kernel/stable.git] / kernel / trace / trace_events_hist.c
CommitLineData
7ef224d1
TZ		 1/*
2 * trace_events_hist - trace event hist triggers
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
15 */
16
17#include <linux/module.h>
18#include <linux/kallsyms.h>
19#include <linux/mutex.h>
20#include <linux/slab.h>
21#include <linux/stacktrace.h>
b2d09103 22#include <linux/rculist.h>
4b147936 23#include <linux/tracefs.h>
7ef224d1
TZ		 24
25#include "tracing_map.h"
26#include "trace.h"
27
4b147936
TZ		 28#define SYNTH_SYSTEM "synthetic"
29#define SYNTH_FIELDS_MAX 16
30
31#define STR_VAR_LEN_MAX 32 /* must be multiple of sizeof(u64) */
32
7ef224d1
TZ		 33struct hist_field;
34
df35d93b
TZ		 35typedef u64 (*hist_field_fn_t) (struct hist_field *field,
36 struct tracing_map_elt *elt,
37 struct ring_buffer_event *rbe,
38 void *event);
7ef224d1 39
5819eadd 40#define HIST_FIELD_OPERANDS_MAX 2
30350d65 41#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
0212e2aa 42#define HIST_ACTIONS_MAX 8
30350d65 43
100719dc
TZ		 44enum field_op_id {
45 FIELD_OP_NONE,
46 FIELD_OP_PLUS,
47 FIELD_OP_MINUS,
48 FIELD_OP_UNARY_MINUS,
49};
50
30350d65
TZ		 51struct hist_var {
52 char *name;
53 struct hist_trigger_data *hist_data;
54 unsigned int idx;
55};
5819eadd 56
7ef224d1
TZ		 57struct hist_field {
58 struct ftrace_event_field *field;
59 unsigned long flags;
60 hist_field_fn_t fn;
61 unsigned int size;
76a3b0c8 62 unsigned int offset;
5819eadd 63 unsigned int is_signed;
19a9facd 64 const char *type;
5819eadd 65 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
b559d003 66 struct hist_trigger_data *hist_data;
30350d65 67 struct hist_var var;
100719dc 68 enum field_op_id operator;
067fe038
TZ		 69 char *system;
70 char *event_name;
100719dc 71 char *name;
067fe038
TZ		 72 unsigned int var_idx;
73 unsigned int var_ref_idx;
74 bool read_once;
7ef224d1
TZ		 75};
76
df35d93b
TZ		 77static u64 hist_field_none(struct hist_field *field,
78 struct tracing_map_elt *elt,
79 struct ring_buffer_event *rbe,
80 void *event)
69a0200c
TZ		 81{
82 return 0;
83}
84
df35d93b
TZ		 85static u64 hist_field_counter(struct hist_field *field,
86 struct tracing_map_elt *elt,
87 struct ring_buffer_event *rbe,
88 void *event)
7ef224d1
TZ		 89{
90 return 1;
91}
92
df35d93b
TZ		 93static u64 hist_field_string(struct hist_field *hist_field,
94 struct tracing_map_elt *elt,
95 struct ring_buffer_event *rbe,
96 void *event)
7ef224d1
TZ		 97{
98 char *addr = (char *)(event + hist_field->field->offset);
99
100 return (u64)(unsigned long)addr;
101}
102
df35d93b
TZ		 103static u64 hist_field_dynstring(struct hist_field *hist_field,
104 struct tracing_map_elt *elt,
105 struct ring_buffer_event *rbe,
106 void *event)
79e577cb
NK		 107{
108 u32 str_item = *(u32 *)(event + hist_field->field->offset);
109 int str_loc = str_item & 0xffff;
110 char *addr = (char *)(event + str_loc);
111
112 return (u64)(unsigned long)addr;
113}
114
df35d93b
TZ		 115static u64 hist_field_pstring(struct hist_field *hist_field,
116 struct tracing_map_elt *elt,
117 struct ring_buffer_event *rbe,
118 void *event)
79e577cb
NK		 119{
120 char **addr = (char **)(event + hist_field->field->offset);
121
122 return (u64)(unsigned long)*addr;
123}
124
df35d93b
TZ		 125static u64 hist_field_log2(struct hist_field *hist_field,
126 struct tracing_map_elt *elt,
127 struct ring_buffer_event *rbe,
128 void *event)
4b94f5b7 129{
5819eadd
TZ		 130 struct hist_field *operand = hist_field->operands[0];
131
df35d93b 132 u64 val = operand->fn(operand, elt, rbe, event);
4b94f5b7
NK		 133
134 return (u64) ilog2(roundup_pow_of_two(val));
135}
136
df35d93b
TZ		 137static u64 hist_field_plus(struct hist_field *hist_field,
138 struct tracing_map_elt *elt,
139 struct ring_buffer_event *rbe,
140 void *event)
100719dc
TZ		 141{
142 struct hist_field *operand1 = hist_field->operands[0];
143 struct hist_field *operand2 = hist_field->operands[1];
144
df35d93b
TZ		 145 u64 val1 = operand1->fn(operand1, elt, rbe, event);
146 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 147
148 return val1 + val2;
149}
150
df35d93b
TZ		 151static u64 hist_field_minus(struct hist_field *hist_field,
152 struct tracing_map_elt *elt,
153 struct ring_buffer_event *rbe,
154 void *event)
100719dc
TZ		 155{
156 struct hist_field *operand1 = hist_field->operands[0];
157 struct hist_field *operand2 = hist_field->operands[1];
158
df35d93b
TZ		 159 u64 val1 = operand1->fn(operand1, elt, rbe, event);
160 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 161
162 return val1 - val2;
163}
164
df35d93b
TZ		 165static u64 hist_field_unary_minus(struct hist_field *hist_field,
166 struct tracing_map_elt *elt,
167 struct ring_buffer_event *rbe,
168 void *event)
100719dc
TZ		 169{
170 struct hist_field *operand = hist_field->operands[0];
171
df35d93b 172 s64 sval = (s64)operand->fn(operand, elt, rbe, event);
100719dc
TZ		 173 u64 val = (u64)-sval;
174
175 return val;
176}
177
7ef224d1 178#define DEFINE_HIST_FIELD_FN(type) \
fbd302cb 179 static u64 hist_field_##type(struct hist_field *hist_field, \
df35d93b
TZ		 180 struct tracing_map_elt *elt, \
181 struct ring_buffer_event *rbe, \
182 void *event) \
7ef224d1
TZ		 183{ \
184 type *addr = (type *)(event + hist_field->field->offset); \
185 \
79e577cb 186 return (u64)(unsigned long)*addr; \
7ef224d1
TZ		 187}
188
189DEFINE_HIST_FIELD_FN(s64);
190DEFINE_HIST_FIELD_FN(u64);
191DEFINE_HIST_FIELD_FN(s32);
192DEFINE_HIST_FIELD_FN(u32);
193DEFINE_HIST_FIELD_FN(s16);
194DEFINE_HIST_FIELD_FN(u16);
195DEFINE_HIST_FIELD_FN(s8);
196DEFINE_HIST_FIELD_FN(u8);
197
198#define for_each_hist_field(i, hist_data) \
199 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
200
201#define for_each_hist_val_field(i, hist_data) \
202 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
203
204#define for_each_hist_key_field(i, hist_data) \
205 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
206
69a0200c
TZ		 207#define HIST_STACKTRACE_DEPTH 16
208#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
209#define HIST_STACKTRACE_SKIP 5
210
7ef224d1 211#define HITCOUNT_IDX 0
69a0200c 212#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
7ef224d1
TZ		 213
214enum hist_field_flags {
0d7a8325
TZ		 215 HIST_FIELD_FL_HITCOUNT = 1 << 0,
216 HIST_FIELD_FL_KEY = 1 << 1,
217 HIST_FIELD_FL_STRING = 1 << 2,
218 HIST_FIELD_FL_HEX = 1 << 3,
219 HIST_FIELD_FL_SYM = 1 << 4,
220 HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
221 HIST_FIELD_FL_EXECNAME = 1 << 6,
222 HIST_FIELD_FL_SYSCALL = 1 << 7,
223 HIST_FIELD_FL_STACKTRACE = 1 << 8,
224 HIST_FIELD_FL_LOG2 = 1 << 9,
ad42febe 225 HIST_FIELD_FL_TIMESTAMP = 1 << 10,
860f9f6b 226 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
30350d65 227 HIST_FIELD_FL_VAR = 1 << 12,
100719dc 228 HIST_FIELD_FL_EXPR = 1 << 13,
067fe038 229 HIST_FIELD_FL_VAR_REF = 1 << 14,
30350d65
TZ		 230};
231
232struct var_defs {
233 unsigned int n_vars;
234 char *name[TRACING_MAP_VARS_MAX];
235 char *expr[TRACING_MAP_VARS_MAX];
7ef224d1
TZ		 236};
237
238struct hist_trigger_attrs {
239 char *keys_str;
f2606835 240 char *vals_str;
e62347d2 241 char *sort_key_str;
5463bfda 242 char *name;
83e99914
TZ		 243 bool pause;
244 bool cont;
e86ae9ba 245 bool clear;
860f9f6b 246 bool ts_in_usecs;
7ef224d1 247 unsigned int map_bits;
30350d65
TZ		 248
249 char *assignment_str[TRACING_MAP_VARS_MAX];
250 unsigned int n_assignments;
251
0212e2aa
TZ		 252 char *action_str[HIST_ACTIONS_MAX];
253 unsigned int n_actions;
254
30350d65 255 struct var_defs var_defs;
7ef224d1
TZ		 256};
257
02205a67
TZ		 258struct field_var {
259 struct hist_field *var;
260 struct hist_field *val;
261};
262
263struct field_var_hist {
264 struct hist_trigger_data *hist_data;
265 char *cmd;
266};
267
7ef224d1 268struct hist_trigger_data {
30350d65 269 struct hist_field *fields[HIST_FIELDS_MAX];
7ef224d1
TZ		 270 unsigned int n_vals;
271 unsigned int n_keys;
272 unsigned int n_fields;
30350d65 273 unsigned int n_vars;
7ef224d1
TZ		 274 unsigned int key_size;
275 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
276 unsigned int n_sort_keys;
277 struct trace_event_file *event_file;
278 struct hist_trigger_attrs *attrs;
279 struct tracing_map *map;
ad42febe 280 bool enable_timestamps;
30350d65 281 bool remove;
067fe038
TZ		 282 struct hist_field *var_refs[TRACING_MAP_VARS_MAX];
283 unsigned int n_var_refs;
0212e2aa
TZ		 284
285 struct action_data *actions[HIST_ACTIONS_MAX];
286 unsigned int n_actions;
02205a67 287
c282a386
TZ		 288 struct hist_field *synth_var_refs[SYNTH_FIELDS_MAX];
289 unsigned int n_synth_var_refs;
02205a67
TZ		 290 struct field_var *field_vars[SYNTH_FIELDS_MAX];
291 unsigned int n_field_vars;
292 unsigned int n_field_var_str;
293 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX];
294 unsigned int n_field_var_hists;
50450603
TZ		 295
296 struct field_var *max_vars[SYNTH_FIELDS_MAX];
297 unsigned int n_max_vars;
298 unsigned int n_max_var_str;
0212e2aa
TZ		 299};
300
4b147936
TZ		 301struct synth_field {
302 char *type;
303 char *name;
304 size_t size;
305 bool is_signed;
306 bool is_string;
307};
308
309struct synth_event {
310 struct list_head list;
311 int ref;
312 char *name;
313 struct synth_field **fields;
314 unsigned int n_fields;
315 unsigned int n_u64;
316 struct trace_event_class class;
317 struct trace_event_call call;
318 struct tracepoint *tp;
319};
320
0212e2aa
TZ		 321struct action_data;
322
323typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
324 struct tracing_map_elt *elt, void *rec,
325 struct ring_buffer_event *rbe,
326 struct action_data *data, u64 *var_ref_vals);
327
328struct action_data {
329 action_fn_t fn;
c282a386
TZ		 330 unsigned int n_params;
331 char *params[SYNTH_FIELDS_MAX];
332
333 union {
334 struct {
335 unsigned int var_ref_idx;
336 char *match_event;
337 char *match_event_system;
338 char *synth_event_name;
339 struct synth_event *synth_event;
340 } onmatch;
50450603
TZ		 341
342 struct {
343 char *var_str;
344 char *fn_name;
345 unsigned int max_var_ref_idx;
346 struct hist_field *max_var;
347 struct hist_field *var;
348 } onmax;
c282a386 349 };
7ef224d1
TZ		 350};
351
4b147936
TZ		 352static LIST_HEAD(synth_event_list);
353static DEFINE_MUTEX(synth_event_mutex);
354
355struct synth_trace_event {
356 struct trace_entry ent;
357 u64 fields[];
358};
359
360static int synth_event_define_fields(struct trace_event_call *call)
361{
362 struct synth_trace_event trace;
363 int offset = offsetof(typeof(trace), fields);
364 struct synth_event *event = call->data;
365 unsigned int i, size, n_u64;
366 char *name, *type;
367 bool is_signed;
368 int ret = 0;
369
370 for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
371 size = event->fields[i]->size;
372 is_signed = event->fields[i]->is_signed;
373 type = event->fields[i]->type;
374 name = event->fields[i]->name;
375 ret = trace_define_field(call, type, name, offset, size,
376 is_signed, FILTER_OTHER);
377 if (ret)
378 break;
379
380 if (event->fields[i]->is_string) {
381 offset += STR_VAR_LEN_MAX;
382 n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
383 } else {
384 offset += sizeof(u64);
385 n_u64++;
386 }
387 }
388
389 event->n_u64 = n_u64;
390
391 return ret;
392}
393
394static bool synth_field_signed(char *type)
395{
396 if (strncmp(type, "u", 1) == 0)
397 return false;
398
399 return true;
400}
401
402static int synth_field_is_string(char *type)
403{
404 if (strstr(type, "char[") != NULL)
405 return true;
406
407 return false;
408}
409
410static int synth_field_string_size(char *type)
411{
412 char buf[4], *end, *start;
413 unsigned int len;
414 int size, err;
415
416 start = strstr(type, "char[");
417 if (start == NULL)
418 return -EINVAL;
419 start += strlen("char[");
420
421 end = strchr(type, ']');
422 if (!end || end < start)
423 return -EINVAL;
424
425 len = end - start;
426 if (len > 3)
427 return -EINVAL;
428
429 strncpy(buf, start, len);
430 buf[len] = '\0';
431
432 err = kstrtouint(buf, 0, &size);
433 if (err)
434 return err;
435
436 if (size > STR_VAR_LEN_MAX)
437 return -EINVAL;
438
439 return size;
440}
441
442static int synth_field_size(char *type)
443{
444 int size = 0;
445
446 if (strcmp(type, "s64") == 0)
447 size = sizeof(s64);
448 else if (strcmp(type, "u64") == 0)
449 size = sizeof(u64);
450 else if (strcmp(type, "s32") == 0)
451 size = sizeof(s32);
452 else if (strcmp(type, "u32") == 0)
453 size = sizeof(u32);
454 else if (strcmp(type, "s16") == 0)
455 size = sizeof(s16);
456 else if (strcmp(type, "u16") == 0)
457 size = sizeof(u16);
458 else if (strcmp(type, "s8") == 0)
459 size = sizeof(s8);
460 else if (strcmp(type, "u8") == 0)
461 size = sizeof(u8);
462 else if (strcmp(type, "char") == 0)
463 size = sizeof(char);
464 else if (strcmp(type, "unsigned char") == 0)
465 size = sizeof(unsigned char);
466 else if (strcmp(type, "int") == 0)
467 size = sizeof(int);
468 else if (strcmp(type, "unsigned int") == 0)
469 size = sizeof(unsigned int);
470 else if (strcmp(type, "long") == 0)
471 size = sizeof(long);
472 else if (strcmp(type, "unsigned long") == 0)
473 size = sizeof(unsigned long);
474 else if (strcmp(type, "pid_t") == 0)
475 size = sizeof(pid_t);
476 else if (synth_field_is_string(type))
477 size = synth_field_string_size(type);
478
479 return size;
480}
481
482static const char *synth_field_fmt(char *type)
483{
484 const char *fmt = "%llu";
485
486 if (strcmp(type, "s64") == 0)
487 fmt = "%lld";
488 else if (strcmp(type, "u64") == 0)
489 fmt = "%llu";
490 else if (strcmp(type, "s32") == 0)
491 fmt = "%d";
492 else if (strcmp(type, "u32") == 0)
493 fmt = "%u";
494 else if (strcmp(type, "s16") == 0)
495 fmt = "%d";
496 else if (strcmp(type, "u16") == 0)
497 fmt = "%u";
498 else if (strcmp(type, "s8") == 0)
499 fmt = "%d";
500 else if (strcmp(type, "u8") == 0)
501 fmt = "%u";
502 else if (strcmp(type, "char") == 0)
503 fmt = "%d";
504 else if (strcmp(type, "unsigned char") == 0)
505 fmt = "%u";
506 else if (strcmp(type, "int") == 0)
507 fmt = "%d";
508 else if (strcmp(type, "unsigned int") == 0)
509 fmt = "%u";
510 else if (strcmp(type, "long") == 0)
511 fmt = "%ld";
512 else if (strcmp(type, "unsigned long") == 0)
513 fmt = "%lu";
514 else if (strcmp(type, "pid_t") == 0)
515 fmt = "%d";
516 else if (synth_field_is_string(type))
517 fmt = "%s";
518
519 return fmt;
520}
521
522static enum print_line_t print_synth_event(struct trace_iterator *iter,
523 int flags,
524 struct trace_event *event)
525{
526 struct trace_array *tr = iter->tr;
527 struct trace_seq *s = &iter->seq;
528 struct synth_trace_event *entry;
529 struct synth_event *se;
530 unsigned int i, n_u64;
531 char print_fmt[32];
532 const char *fmt;
533
534 entry = (struct synth_trace_event *)iter->ent;
535 se = container_of(event, struct synth_event, call.event);
536
537 trace_seq_printf(s, "%s: ", se->name);
538
539 for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
540 if (trace_seq_has_overflowed(s))
541 goto end;
542
543 fmt = synth_field_fmt(se->fields[i]->type);
544
545 /* parameter types */
546 if (tr->trace_flags & TRACE_ITER_VERBOSE)
547 trace_seq_printf(s, "%s ", fmt);
548
549 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
550
551 /* parameter values */
552 if (se->fields[i]->is_string) {
553 trace_seq_printf(s, print_fmt, se->fields[i]->name,
554 (char *)&entry->fields[n_u64],
555 i == se->n_fields - 1 ? "" : " ");
556 n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
557 } else {
558 trace_seq_printf(s, print_fmt, se->fields[i]->name,
559 entry->fields[n_u64],
560 i == se->n_fields - 1 ? "" : " ");
561 n_u64++;
562 }
563 }
564end:
565 trace_seq_putc(s, '\n');
566
567 return trace_handle_return(s);
568}
569
570static struct trace_event_functions synth_event_funcs = {
571 .trace = print_synth_event
572};
573
574static notrace void trace_event_raw_event_synth(void *__data,
575 u64 *var_ref_vals,
576 unsigned int var_ref_idx)
577{
578 struct trace_event_file *trace_file = __data;
579 struct synth_trace_event *entry;
580 struct trace_event_buffer fbuffer;
581 struct synth_event *event;
582 unsigned int i, n_u64;
583 int fields_size = 0;
584
585 event = trace_file->event_call->data;
586
587 if (trace_trigger_soft_disabled(trace_file))
588 return;
589
590 fields_size = event->n_u64 * sizeof(u64);
591
592 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
593 sizeof(*entry) + fields_size);
594 if (!entry)
595 return;
596
597 for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
598 if (event->fields[i]->is_string) {
599 char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
600 char *str_field = (char *)&entry->fields[n_u64];
601
602 strncpy(str_field, str_val, STR_VAR_LEN_MAX);
603 n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
604 } else {
605 entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
606 n_u64++;
607 }
608 }
609
610 trace_event_buffer_commit(&fbuffer);
611}
612
613static void free_synth_event_print_fmt(struct trace_event_call *call)
614{
615 if (call) {
616 kfree(call->print_fmt);
617 call->print_fmt = NULL;
618 }
619}
620
621static int __set_synth_event_print_fmt(struct synth_event *event,
622 char *buf, int len)
623{
624 const char *fmt;
625 int pos = 0;
626 int i;
627
628 /* When len=0, we just calculate the needed length */
629#define LEN_OR_ZERO (len ? len - pos : 0)
630
631 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
632 for (i = 0; i < event->n_fields; i++) {
633 fmt = synth_field_fmt(event->fields[i]->type);
634 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
635 event->fields[i]->name, fmt,
636 i == event->n_fields - 1 ? "" : ", ");
637 }
638 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
639
640 for (i = 0; i < event->n_fields; i++) {
641 pos += snprintf(buf + pos, LEN_OR_ZERO,
642 ", REC->%s", event->fields[i]->name);
643 }
644
645#undef LEN_OR_ZERO
646
647 /* return the length of print_fmt */
648 return pos;
649}
650
651static int set_synth_event_print_fmt(struct trace_event_call *call)
652{
653 struct synth_event *event = call->data;
654 char *print_fmt;
655 int len;
656
657 /* First: called with 0 length to calculate the needed length */
658 len = __set_synth_event_print_fmt(event, NULL, 0);
659
660 print_fmt = kmalloc(len + 1, GFP_KERNEL);
661 if (!print_fmt)
662 return -ENOMEM;
663
664 /* Second: actually write the @print_fmt */
665 __set_synth_event_print_fmt(event, print_fmt, len + 1);
666 call->print_fmt = print_fmt;
667
668 return 0;
669}
670
671static void free_synth_field(struct synth_field *field)
672{
673 kfree(field->type);
674 kfree(field->name);
675 kfree(field);
676}
677
678static struct synth_field *parse_synth_field(char *field_type,
679 char *field_name)
680{
681 struct synth_field *field;
682 int len, ret = 0;
683 char *array;
684
685 if (field_type[0] == ';')
686 field_type++;
687
688 len = strlen(field_name);
689 if (field_name[len - 1] == ';')
690 field_name[len - 1] = '\0';
691
692 field = kzalloc(sizeof(*field), GFP_KERNEL);
693 if (!field)
694 return ERR_PTR(-ENOMEM);
695
696 len = strlen(field_type) + 1;
697 array = strchr(field_name, '[');
698 if (array)
699 len += strlen(array);
700 field->type = kzalloc(len, GFP_KERNEL);
701 if (!field->type) {
702 ret = -ENOMEM;
703 goto free;
704 }
705 strcat(field->type, field_type);
706 if (array) {
707 strcat(field->type, array);
708 *array = '\0';
709 }
710
711 field->size = synth_field_size(field->type);
712 if (!field->size) {
713 ret = -EINVAL;
714 goto free;
715 }
716
717 if (synth_field_is_string(field->type))
718 field->is_string = true;
719
720 field->is_signed = synth_field_signed(field->type);
721
722 field->name = kstrdup(field_name, GFP_KERNEL);
723 if (!field->name) {
724 ret = -ENOMEM;
725 goto free;
726 }
727 out:
728 return field;
729 free:
730 free_synth_field(field);
731 field = ERR_PTR(ret);
732 goto out;
733}
734
735static void free_synth_tracepoint(struct tracepoint *tp)
736{
737 if (!tp)
738 return;
739
740 kfree(tp->name);
741 kfree(tp);
742}
743
744static struct tracepoint *alloc_synth_tracepoint(char *name)
745{
746 struct tracepoint *tp;
747
748 tp = kzalloc(sizeof(*tp), GFP_KERNEL);
749 if (!tp)
750 return ERR_PTR(-ENOMEM);
751
752 tp->name = kstrdup(name, GFP_KERNEL);
753 if (!tp->name) {
754 kfree(tp);
755 return ERR_PTR(-ENOMEM);
756 }
757
758 return tp;
759}
760
761typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
762 unsigned int var_ref_idx);
763
764static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
765 unsigned int var_ref_idx)
766{
767 struct tracepoint *tp = event->tp;
768
769 if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
770 struct tracepoint_func *probe_func_ptr;
771 synth_probe_func_t probe_func;
772 void *__data;
773
774 if (!(cpu_online(raw_smp_processor_id())))
775 return;
776
777 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
778 if (probe_func_ptr) {
779 do {
780 probe_func = probe_func_ptr->func;
781 __data = probe_func_ptr->data;
782 probe_func(__data, var_ref_vals, var_ref_idx);
783 } while ((++probe_func_ptr)->func);
784 }
785 }
786}
787
788static struct synth_event *find_synth_event(const char *name)
789{
790 struct synth_event *event;
791
792 list_for_each_entry(event, &synth_event_list, list) {
793 if (strcmp(event->name, name) == 0)
794 return event;
795 }
796
797 return NULL;
798}
799
800static int register_synth_event(struct synth_event *event)
801{
802 struct trace_event_call *call = &event->call;
803 int ret = 0;
804
805 event->call.class = &event->class;
806 event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
807 if (!event->class.system) {
808 ret = -ENOMEM;
809 goto out;
810 }
811
812 event->tp = alloc_synth_tracepoint(event->name);
813 if (IS_ERR(event->tp)) {
814 ret = PTR_ERR(event->tp);
815 event->tp = NULL;
816 goto out;
817 }
818
819 INIT_LIST_HEAD(&call->class->fields);
820 call->event.funcs = &synth_event_funcs;
821 call->class->define_fields = synth_event_define_fields;
822
823 ret = register_trace_event(&call->event);
824 if (!ret) {
825 ret = -ENODEV;
826 goto out;
827 }
828 call->flags = TRACE_EVENT_FL_TRACEPOINT;
829 call->class->reg = trace_event_reg;
830 call->class->probe = trace_event_raw_event_synth;
831 call->data = event;
832 call->tp = event->tp;
833
834 ret = trace_add_event_call(call);
835 if (ret) {
836 pr_warn("Failed to register synthetic event: %s\n",
837 trace_event_name(call));
838 goto err;
839 }
840
841 ret = set_synth_event_print_fmt(call);
842 if (ret < 0) {
843 trace_remove_event_call(call);
844 goto err;
845 }
846 out:
847 return ret;
848 err:
849 unregister_trace_event(&call->event);
850 goto out;
851}
852
853static int unregister_synth_event(struct synth_event *event)
854{
855 struct trace_event_call *call = &event->call;
856 int ret;
857
858 ret = trace_remove_event_call(call);
859
860 return ret;
861}
862
863static void free_synth_event(struct synth_event *event)
864{
865 unsigned int i;
866
867 if (!event)
868 return;
869
870 for (i = 0; i < event->n_fields; i++)
871 free_synth_field(event->fields[i]);
872
873 kfree(event->fields);
874 kfree(event->name);
875 kfree(event->class.system);
876 free_synth_tracepoint(event->tp);
877 free_synth_event_print_fmt(&event->call);
878 kfree(event);
879}
880
881static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
882 struct synth_field **fields)
883{
884 struct synth_event *event;
885 unsigned int i;
886
887 event = kzalloc(sizeof(*event), GFP_KERNEL);
888 if (!event) {
889 event = ERR_PTR(-ENOMEM);
890 goto out;
891 }
892
893 event->name = kstrdup(event_name, GFP_KERNEL);
894 if (!event->name) {
895 kfree(event);
896 event = ERR_PTR(-ENOMEM);
897 goto out;
898 }
899
900 event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
901 if (!event->fields) {
902 free_synth_event(event);
903 event = ERR_PTR(-ENOMEM);
904 goto out;
905 }
906
907 for (i = 0; i < n_fields; i++)
908 event->fields[i] = fields[i];
909
910 event->n_fields = n_fields;
911 out:
912 return event;
913}
914
c282a386
TZ		 915static void action_trace(struct hist_trigger_data *hist_data,
916 struct tracing_map_elt *elt, void *rec,
917 struct ring_buffer_event *rbe,
918 struct action_data *data, u64 *var_ref_vals)
919{
920 struct synth_event *event = data->onmatch.synth_event;
921
922 trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
923}
924
925struct hist_var_data {
926 struct list_head list;
927 struct hist_trigger_data *hist_data;
928};
929
4b147936
TZ		 930static void add_or_delete_synth_event(struct synth_event *event, int delete)
931{
932 if (delete)
933 free_synth_event(event);
934 else {
935 mutex_lock(&synth_event_mutex);
936 if (!find_synth_event(event->name))
937 list_add(&event->list, &synth_event_list);
938 else
939 free_synth_event(event);
940 mutex_unlock(&synth_event_mutex);
941 }
942}
943
944static int create_synth_event(int argc, char **argv)
945{
946 struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
947 struct synth_event *event = NULL;
948 bool delete_event = false;
949 int i, n_fields = 0, ret = 0;
950 char *name;
951
952 mutex_lock(&synth_event_mutex);
953
954 /*
955 * Argument syntax:
956 * - Add synthetic event: <event_name> field[;field] ...
957 * - Remove synthetic event: !<event_name> field[;field] ...
958 * where 'field' = type field_name
959 */
960 if (argc < 1) {
961 ret = -EINVAL;
962 goto out;
963 }
964
965 name = argv[0];
966 if (name[0] == '!') {
967 delete_event = true;
968 name++;
969 }
970
971 event = find_synth_event(name);
972 if (event) {
973 if (delete_event) {
974 if (event->ref) {
975 event = NULL;
976 ret = -EBUSY;
977 goto out;
978 }
979 list_del(&event->list);
980 goto out;
981 }
982 event = NULL;
983 ret = -EEXIST;
984 goto out;
985 } else if (delete_event)
986 goto out;
987
988 if (argc < 2) {
989 ret = -EINVAL;
990 goto out;
991 }
992
993 for (i = 1; i < argc - 1; i++) {
994 if (strcmp(argv[i], ";") == 0)
995 continue;
996 if (n_fields == SYNTH_FIELDS_MAX) {
997 ret = -EINVAL;
998 goto err;
999 }
1000
1001 field = parse_synth_field(argv[i], argv[i + 1]);
1002 if (IS_ERR(field)) {
1003 ret = PTR_ERR(field);
1004 goto err;
1005 }
1006 fields[n_fields] = field;
1007 i++; n_fields++;
1008 }
1009
1010 if (i < argc) {
1011 ret = -EINVAL;
1012 goto err;
1013 }
1014
1015 event = alloc_synth_event(name, n_fields, fields);
1016 if (IS_ERR(event)) {
1017 ret = PTR_ERR(event);
1018 event = NULL;
1019 goto err;
1020 }
1021 out:
1022 mutex_unlock(&synth_event_mutex);
1023
1024 if (event) {
1025 if (delete_event) {
1026 ret = unregister_synth_event(event);
1027 add_or_delete_synth_event(event, !ret);
1028 } else {
1029 ret = register_synth_event(event);
1030 add_or_delete_synth_event(event, ret);
1031 }
1032 }
1033
1034 return ret;
1035 err:
1036 mutex_unlock(&synth_event_mutex);
1037
1038 for (i = 0; i < n_fields; i++)
1039 free_synth_field(fields[i]);
1040 free_synth_event(event);
1041
1042 return ret;
1043}
1044
1045static int release_all_synth_events(void)
1046{
1047 struct list_head release_events;
1048 struct synth_event *event, *e;
1049 int ret = 0;
1050
1051 INIT_LIST_HEAD(&release_events);
1052
1053 mutex_lock(&synth_event_mutex);
1054
1055 list_for_each_entry(event, &synth_event_list, list) {
1056 if (event->ref) {
1057 mutex_unlock(&synth_event_mutex);
1058 return -EBUSY;
1059 }
1060 }
1061
1062 list_splice_init(&event->list, &release_events);
1063
1064 mutex_unlock(&synth_event_mutex);
1065
1066 list_for_each_entry_safe(event, e, &release_events, list) {
1067 list_del(&event->list);
1068
1069 ret = unregister_synth_event(event);
1070 add_or_delete_synth_event(event, !ret);
1071 }
1072
1073 return ret;
1074}
1075
1076
1077static void *synth_events_seq_start(struct seq_file *m, loff_t *pos)
1078{
1079 mutex_lock(&synth_event_mutex);
1080
1081 return seq_list_start(&synth_event_list, *pos);
1082}
1083
1084static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos)
1085{
1086 return seq_list_next(v, &synth_event_list, pos);
1087}
1088
1089static void synth_events_seq_stop(struct seq_file *m, void *v)
1090{
1091 mutex_unlock(&synth_event_mutex);
1092}
1093
1094static int synth_events_seq_show(struct seq_file *m, void *v)
1095{
1096 struct synth_field *field;
1097 struct synth_event *event = v;
1098 unsigned int i;
1099
1100 seq_printf(m, "%s\t", event->name);
1101
1102 for (i = 0; i < event->n_fields; i++) {
1103 field = event->fields[i];
1104
1105 /* parameter values */
1106 seq_printf(m, "%s %s%s", field->type, field->name,
1107 i == event->n_fields - 1 ? "" : "; ");
1108 }
1109
1110 seq_putc(m, '\n');
1111
1112 return 0;
1113}
1114
1115static const struct seq_operations synth_events_seq_op = {
1116 .start = synth_events_seq_start,
1117 .next = synth_events_seq_next,
1118 .stop = synth_events_seq_stop,
1119 .show = synth_events_seq_show
1120};
1121
1122static int synth_events_open(struct inode *inode, struct file *file)
1123{
1124 int ret;
1125
1126 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1127 ret = release_all_synth_events();
1128 if (ret < 0)
1129 return ret;
1130 }
1131
1132 return seq_open(file, &synth_events_seq_op);
1133}
1134
1135static ssize_t synth_events_write(struct file *file,
1136 const char __user *buffer,
1137 size_t count, loff_t *ppos)
1138{
1139 return trace_parse_run_command(file, buffer, count, ppos,
1140 create_synth_event);
1141}
1142
1143static const struct file_operations synth_events_fops = {
1144 .open = synth_events_open,
1145 .write = synth_events_write,
1146 .read = seq_read,
1147 .llseek = seq_lseek,
1148 .release = seq_release,
1149};
1150
df35d93b
TZ		 1151static u64 hist_field_timestamp(struct hist_field *hist_field,
1152 struct tracing_map_elt *elt,
1153 struct ring_buffer_event *rbe,
1154 void *event)
860f9f6b
TZ		 1155{
1156 struct hist_trigger_data *hist_data = hist_field->hist_data;
1157 struct trace_array *tr = hist_data->event_file->tr;
1158
1159 u64 ts = ring_buffer_event_time_stamp(rbe);
1160
1161 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1162 ts = ns2usecs(ts);
1163
1164 return ts;
1165}
1166
067fe038
TZ		 1167static struct hist_field *
1168check_field_for_var_ref(struct hist_field *hist_field,
1169 struct hist_trigger_data *var_data,
1170 unsigned int var_idx)
1171{
1172 struct hist_field *found = NULL;
1173
1174 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1175 if (hist_field->var.idx == var_idx &&
1176 hist_field->var.hist_data == var_data) {
1177 found = hist_field;
1178 }
1179 }
1180
1181 return found;
1182}
1183
1184static struct hist_field *
1185check_field_for_var_refs(struct hist_trigger_data *hist_data,
1186 struct hist_field *hist_field,
1187 struct hist_trigger_data *var_data,
1188 unsigned int var_idx,
1189 unsigned int level)
1190{
1191 struct hist_field *found = NULL;
1192 unsigned int i;
1193
1194 if (level > 3)
1195 return found;
1196
1197 if (!hist_field)
1198 return found;
1199
1200 found = check_field_for_var_ref(hist_field, var_data, var_idx);
1201 if (found)
1202 return found;
1203
1204 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1205 struct hist_field *operand;
1206
1207 operand = hist_field->operands[i];
1208 found = check_field_for_var_refs(hist_data, operand, var_data,
1209 var_idx, level + 1);
1210 if (found)
1211 return found;
1212 }
1213
1214 return found;
1215}
1216
1217static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1218 struct hist_trigger_data *var_data,
1219 unsigned int var_idx)
1220{
1221 struct hist_field *hist_field, *found = NULL;
1222 unsigned int i;
1223
1224 for_each_hist_field(i, hist_data) {
1225 hist_field = hist_data->fields[i];
1226 found = check_field_for_var_refs(hist_data, hist_field,
1227 var_data, var_idx, 0);
1228 if (found)
1229 return found;
1230 }
1231
c282a386
TZ		 1232 for (i = 0; i < hist_data->n_synth_var_refs; i++) {
1233 hist_field = hist_data->synth_var_refs[i];
1234 found = check_field_for_var_refs(hist_data, hist_field,
1235 var_data, var_idx, 0);
1236 if (found)
1237 return found;
1238 }
1239
067fe038
TZ		 1240 return found;
1241}
1242
1243static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1244 unsigned int var_idx)
1245{
1246 struct trace_array *tr = hist_data->event_file->tr;
1247 struct hist_field *found = NULL;
1248 struct hist_var_data *var_data;
1249
1250 list_for_each_entry(var_data, &tr->hist_vars, list) {
1251 if (var_data->hist_data == hist_data)
1252 continue;
1253 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1254 if (found)
1255 break;
1256 }
1257
1258 return found;
1259}
1260
1261static bool check_var_refs(struct hist_trigger_data *hist_data)
1262{
1263 struct hist_field *field;
1264 bool found = false;
1265 int i;
1266
1267 for_each_hist_field(i, hist_data) {
1268 field = hist_data->fields[i];
1269 if (field && field->flags & HIST_FIELD_FL_VAR) {
1270 if (find_any_var_ref(hist_data, field->var.idx)) {
1271 found = true;
1272 break;
1273 }
1274 }
1275 }
1276
1277 return found;
1278}
1279
1280static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1281{
1282 struct trace_array *tr = hist_data->event_file->tr;
1283 struct hist_var_data *var_data, *found = NULL;
1284
1285 list_for_each_entry(var_data, &tr->hist_vars, list) {
1286 if (var_data->hist_data == hist_data) {
1287 found = var_data;
1288 break;
1289 }
1290 }
1291
1292 return found;
1293}
1294
1295static bool field_has_hist_vars(struct hist_field *hist_field,
1296 unsigned int level)
1297{
1298 int i;
1299
1300 if (level > 3)
1301 return false;
1302
1303 if (!hist_field)
1304 return false;
1305
1306 if (hist_field->flags & HIST_FIELD_FL_VAR ||
1307 hist_field->flags & HIST_FIELD_FL_VAR_REF)
1308 return true;
1309
1310 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1311 struct hist_field *operand;
1312
1313 operand = hist_field->operands[i];
1314 if (field_has_hist_vars(operand, level + 1))
1315 return true;
1316 }
1317
1318 return false;
1319}
1320
1321static bool has_hist_vars(struct hist_trigger_data *hist_data)
1322{
1323 struct hist_field *hist_field;
1324 int i;
1325
1326 for_each_hist_field(i, hist_data) {
1327 hist_field = hist_data->fields[i];
1328 if (field_has_hist_vars(hist_field, 0))
1329 return true;
1330 }
1331
1332 return false;
1333}
1334
1335static int save_hist_vars(struct hist_trigger_data *hist_data)
1336{
1337 struct trace_array *tr = hist_data->event_file->tr;
1338 struct hist_var_data *var_data;
1339
1340 var_data = find_hist_vars(hist_data);
1341 if (var_data)
1342 return 0;
1343
1344 if (trace_array_get(tr) < 0)
1345 return -ENODEV;
1346
1347 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1348 if (!var_data) {
1349 trace_array_put(tr);
1350 return -ENOMEM;
1351 }
1352
1353 var_data->hist_data = hist_data;
1354 list_add(&var_data->list, &tr->hist_vars);
1355
1356 return 0;
1357}
1358
1359static void remove_hist_vars(struct hist_trigger_data *hist_data)
1360{
1361 struct trace_array *tr = hist_data->event_file->tr;
1362 struct hist_var_data *var_data;
1363
1364 var_data = find_hist_vars(hist_data);
1365 if (!var_data)
1366 return;
1367
1368 if (WARN_ON(check_var_refs(hist_data)))
1369 return;
1370
1371 list_del(&var_data->list);
1372
1373 kfree(var_data);
1374
1375 trace_array_put(tr);
1376}
1377
30350d65
TZ		 1378static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1379 const char *var_name)
1380{
1381 struct hist_field *hist_field, *found = NULL;
1382 int i;
1383
1384 for_each_hist_field(i, hist_data) {
1385 hist_field = hist_data->fields[i];
1386 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1387 strcmp(hist_field->var.name, var_name) == 0) {
1388 found = hist_field;
1389 break;
1390 }
1391 }
1392
1393 return found;
1394}
1395
1396static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1397 struct trace_event_file *file,
1398 const char *var_name)
1399{
1400 struct hist_trigger_data *test_data;
1401 struct event_trigger_data *test;
1402 struct hist_field *hist_field;
1403
1404 hist_field = find_var_field(hist_data, var_name);
1405 if (hist_field)
1406 return hist_field;
1407
1408 list_for_each_entry_rcu(test, &file->triggers, list) {
1409 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1410 test_data = test->private_data;
1411 hist_field = find_var_field(test_data, var_name);
1412 if (hist_field)
1413 return hist_field;
1414 }
1415 }
1416
1417 return NULL;
1418}
1419
067fe038
TZ		 1420static struct trace_event_file *find_var_file(struct trace_array *tr,
1421 char *system,
1422 char *event_name,
1423 char *var_name)
1424{
1425 struct hist_trigger_data *var_hist_data;
1426 struct hist_var_data *var_data;
1427 struct trace_event_file *file, *found = NULL;
1428
1429 if (system)
1430 return find_event_file(tr, system, event_name);
1431
1432 list_for_each_entry(var_data, &tr->hist_vars, list) {
1433 var_hist_data = var_data->hist_data;
1434 file = var_hist_data->event_file;
1435 if (file == found)
1436 continue;
1437
1438 if (find_var_field(var_hist_data, var_name)) {
1439 if (found)
1440 return NULL;
1441
1442 found = file;
1443 }
1444 }
1445
1446 return found;
1447}
1448
1449static struct hist_field *find_file_var(struct trace_event_file *file,
1450 const char *var_name)
1451{
1452 struct hist_trigger_data *test_data;
1453 struct event_trigger_data *test;
1454 struct hist_field *hist_field;
1455
1456 list_for_each_entry_rcu(test, &file->triggers, list) {
1457 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1458 test_data = test->private_data;
1459 hist_field = find_var_field(test_data, var_name);
1460 if (hist_field)
1461 return hist_field;
1462 }
1463 }
1464
1465 return NULL;
1466}
1467
c282a386
TZ		 1468static struct hist_field *
1469find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1470{
1471 struct trace_array *tr = hist_data->event_file->tr;
1472 struct hist_field *hist_field, *found = NULL;
1473 struct trace_event_file *file;
1474 unsigned int i;
1475
1476 for (i = 0; i < hist_data->n_actions; i++) {
1477 struct action_data *data = hist_data->actions[i];
1478
1479 if (data->fn == action_trace) {
1480 char *system = data->onmatch.match_event_system;
1481 char *event_name = data->onmatch.match_event;
1482
1483 file = find_var_file(tr, system, event_name, var_name);
1484 if (!file)
1485 continue;
1486 hist_field = find_file_var(file, var_name);
1487 if (hist_field) {
1488 if (found) {
1489 return ERR_PTR(-EINVAL);
1490 }
1491
1492 found = hist_field;
1493 }
1494 }
1495 }
1496 return found;
1497}
1498
067fe038
TZ		 1499static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1500 char *system,
1501 char *event_name,
1502 char *var_name)
1503{
1504 struct trace_array *tr = hist_data->event_file->tr;
1505 struct hist_field *hist_field = NULL;
1506 struct trace_event_file *file;
1507
c282a386
TZ		 1508 if (!system || !event_name) {
1509 hist_field = find_match_var(hist_data, var_name);
1510 if (IS_ERR(hist_field))
1511 return NULL;
1512 if (hist_field)
1513 return hist_field;
1514 }
1515
067fe038
TZ		 1516 file = find_var_file(tr, system, event_name, var_name);
1517 if (!file)
1518 return NULL;
1519
1520 hist_field = find_file_var(file, var_name);
1521
1522 return hist_field;
1523}
1524
af6a29bc
TZ		 1525struct hist_elt_data {
1526 char *comm;
067fe038 1527 u64 *var_ref_vals;
02205a67 1528 char *field_var_str[SYNTH_FIELDS_MAX];
af6a29bc
TZ		 1529};
1530
067fe038
TZ		 1531static u64 hist_field_var_ref(struct hist_field *hist_field,
1532 struct tracing_map_elt *elt,
1533 struct ring_buffer_event *rbe,
1534 void *event)
1535{
1536 struct hist_elt_data *elt_data;
1537 u64 var_val = 0;
1538
1539 elt_data = elt->private_data;
1540 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1541
1542 return var_val;
1543}
1544
1545static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1546 u64 *var_ref_vals, bool self)
1547{
1548 struct hist_trigger_data *var_data;
1549 struct tracing_map_elt *var_elt;
1550 struct hist_field *hist_field;
1551 unsigned int i, var_idx;
1552 bool resolved = true;
1553 u64 var_val = 0;
1554
1555 for (i = 0; i < hist_data->n_var_refs; i++) {
1556 hist_field = hist_data->var_refs[i];
1557 var_idx = hist_field->var.idx;
1558 var_data = hist_field->var.hist_data;
1559
1560 if (var_data == NULL) {
1561 resolved = false;
1562 break;
1563 }
1564
1565 if ((self && var_data != hist_data) ||
1566 (!self && var_data == hist_data))
1567 continue;
1568
1569 var_elt = tracing_map_lookup(var_data->map, key);
1570 if (!var_elt) {
1571 resolved = false;
1572 break;
1573 }
1574
1575 if (!tracing_map_var_set(var_elt, var_idx)) {
1576 resolved = false;
1577 break;
1578 }
1579
1580 if (self || !hist_field->read_once)
1581 var_val = tracing_map_read_var(var_elt, var_idx);
1582 else
1583 var_val = tracing_map_read_var_once(var_elt, var_idx);
1584
1585 var_ref_vals[i] = var_val;
1586 }
1587
1588 return resolved;
1589}
1590
85013256
TZ		 1591static const char *hist_field_name(struct hist_field *field,
1592 unsigned int level)
1593{
1594 const char *field_name = "";
1595
1596 if (level > 1)
1597 return field_name;
1598
1599 if (field->field)
1600 field_name = field->field->name;
5819eadd
TZ		 1601 else if (field->flags & HIST_FIELD_FL_LOG2)
1602 field_name = hist_field_name(field->operands[0], ++level);
ad42febe
TZ		 1603 else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1604 field_name = "common_timestamp";
067fe038
TZ		 1605 else if (field->flags & HIST_FIELD_FL_EXPR ||
1606 field->flags & HIST_FIELD_FL_VAR_REF) {
1607 if (field->system) {
1608 static char full_name[MAX_FILTER_STR_VAL];
1609
1610 strcat(full_name, field->system);
1611 strcat(full_name, ".");
1612 strcat(full_name, field->event_name);
1613 strcat(full_name, ".");
1614 strcat(full_name, field->name);
1615 field_name = full_name;
1616 } else
1617 field_name = field->name;
1618 }
85013256
TZ		 1619
1620 if (field_name == NULL)
1621 field_name = "";
1622
1623 return field_name;
1624}
1625
7ef224d1
TZ		 1626static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1627{
1628 hist_field_fn_t fn = NULL;
1629
1630 switch (field_size) {
1631 case 8:
1632 if (field_is_signed)
1633 fn = hist_field_s64;
1634 else
1635 fn = hist_field_u64;
1636 break;
1637 case 4:
1638 if (field_is_signed)
1639 fn = hist_field_s32;
1640 else
1641 fn = hist_field_u32;
1642 break;
1643 case 2:
1644 if (field_is_signed)
1645 fn = hist_field_s16;
1646 else
1647 fn = hist_field_u16;
1648 break;
1649 case 1:
1650 if (field_is_signed)
1651 fn = hist_field_s8;
1652 else
1653 fn = hist_field_u8;
1654 break;
1655 }
1656
1657 return fn;
1658}
1659
1660static int parse_map_size(char *str)
1661{
1662 unsigned long size, map_bits;
1663 int ret;
1664
1665 strsep(&str, "=");
1666 if (!str) {
1667 ret = -EINVAL;
1668 goto out;
1669 }
1670
1671 ret = kstrtoul(str, 0, &size);
1672 if (ret)
1673 goto out;
1674
1675 map_bits = ilog2(roundup_pow_of_two(size));
1676 if (map_bits < TRACING_MAP_BITS_MIN ||
1677 map_bits > TRACING_MAP_BITS_MAX)
1678 ret = -EINVAL;
1679 else
1680 ret = map_bits;
1681 out:
1682 return ret;
1683}
1684
1685static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1686{
30350d65
TZ		 1687 unsigned int i;
1688
7ef224d1
TZ		 1689 if (!attrs)
1690 return;
1691
30350d65
TZ		 1692 for (i = 0; i < attrs->n_assignments; i++)
1693 kfree(attrs->assignment_str[i]);
1694
0212e2aa
TZ		 1695 for (i = 0; i < attrs->n_actions; i++)
1696 kfree(attrs->action_str[i]);
1697
5463bfda 1698 kfree(attrs->name);
e62347d2 1699 kfree(attrs->sort_key_str);
7ef224d1 1700 kfree(attrs->keys_str);
f2606835 1701 kfree(attrs->vals_str);
7ef224d1
TZ		 1702 kfree(attrs);
1703}
1704
0212e2aa
TZ		 1705static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1706{
c282a386 1707 int ret = -EINVAL;
0212e2aa
TZ		 1708
1709 if (attrs->n_actions >= HIST_ACTIONS_MAX)
1710 return ret;
1711
50450603
TZ		 1712 if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) ||
1713 (strncmp(str, "onmax(", strlen("onmax(")) == 0)) {
c282a386
TZ		 1714 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1715 if (!attrs->action_str[attrs->n_actions]) {
1716 ret = -ENOMEM;
1717 return ret;
1718 }
1719 attrs->n_actions++;
1720 ret = 0;
1721 }
1722
0212e2aa
TZ		 1723 return ret;
1724}
1725
9b1ae035
TZ		 1726static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1727{
1728 int ret = 0;
1729
1730 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
1731 (strncmp(str, "keys=", strlen("keys=")) == 0)) {
1732 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1733 if (!attrs->keys_str) {
1734 ret = -ENOMEM;
1735 goto out;
1736 }
1737 } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
1738 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
1739 (strncmp(str, "values=", strlen("values=")) == 0)) {
1740 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1741 if (!attrs->vals_str) {
1742 ret = -ENOMEM;
1743 goto out;
1744 }
1745 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
1746 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1747 if (!attrs->sort_key_str) {
1748 ret = -ENOMEM;
1749 goto out;
1750 }
1751 } else if (strncmp(str, "name=", strlen("name=")) == 0) {
1752 attrs->name = kstrdup(str, GFP_KERNEL);
1753 if (!attrs->name) {
1754 ret = -ENOMEM;
1755 goto out;
1756 }
1757 } else if (strncmp(str, "size=", strlen("size=")) == 0) {
1758 int map_bits = parse_map_size(str);
1759
1760 if (map_bits < 0) {
1761 ret = map_bits;
1762 goto out;
1763 }
1764 attrs->map_bits = map_bits;
30350d65
TZ		 1765 } else {
1766 char *assignment;
1767
1768 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1769 ret = -EINVAL;
1770 goto out;
1771 }
1772
1773 assignment = kstrdup(str, GFP_KERNEL);
1774 if (!assignment) {
1775 ret = -ENOMEM;
1776 goto out;
1777 }
1778
1779 attrs->assignment_str[attrs->n_assignments++] = assignment;
1780 }
9b1ae035
TZ		 1781 out:
1782 return ret;
1783}
1784
7ef224d1
TZ		 1785static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1786{
1787 struct hist_trigger_attrs *attrs;
1788 int ret = 0;
1789
1790 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1791 if (!attrs)
1792 return ERR_PTR(-ENOMEM);
1793
1794 while (trigger_str) {
1795 char *str = strsep(&trigger_str, ":");
1796
9b1ae035
TZ		 1797 if (strchr(str, '=')) {
1798 ret = parse_assignment(str, attrs);
1799 if (ret)
1800 goto free;
1801 } else if (strcmp(str, "pause") == 0)
83e99914
TZ		 1802 attrs->pause = true;
1803 else if ((strcmp(str, "cont") == 0) ||
1804 (strcmp(str, "continue") == 0))
1805 attrs->cont = true;
e86ae9ba
TZ		 1806 else if (strcmp(str, "clear") == 0)
1807 attrs->clear = true;
9b1ae035 1808 else {
0212e2aa
TZ		 1809 ret = parse_action(str, attrs);
1810 if (ret)
1811 goto free;
7ef224d1
TZ		 1812 }
1813 }
1814
1815 if (!attrs->keys_str) {
1816 ret = -EINVAL;
1817 goto free;
1818 }
1819
1820 return attrs;
1821 free:
1822 destroy_hist_trigger_attrs(attrs);
1823
1824 return ERR_PTR(ret);
1825}
1826
6b4827ad
TZ		 1827static inline void save_comm(char *comm, struct task_struct *task)
1828{
1829 if (!task->pid) {
1830 strcpy(comm, "<idle>");
1831 return;
1832 }
1833
1834 if (WARN_ON_ONCE(task->pid < 0)) {
1835 strcpy(comm, "<XXX>");
1836 return;
1837 }
1838
1839 memcpy(comm, task->comm, TASK_COMM_LEN);
1840}
1841
af6a29bc
TZ		 1842static void hist_elt_data_free(struct hist_elt_data *elt_data)
1843{
02205a67
TZ		 1844 unsigned int i;
1845
1846 for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1847 kfree(elt_data->field_var_str[i]);
1848
af6a29bc
TZ		 1849 kfree(elt_data->comm);
1850 kfree(elt_data);
1851}
1852
1853static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
6b4827ad 1854{
af6a29bc
TZ		 1855 struct hist_elt_data *elt_data = elt->private_data;
1856
1857 hist_elt_data_free(elt_data);
6b4827ad
TZ		 1858}
1859
af6a29bc 1860static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
6b4827ad
TZ		 1861{
1862 struct hist_trigger_data *hist_data = elt->map->private_data;
af6a29bc
TZ		 1863 unsigned int size = TASK_COMM_LEN;
1864 struct hist_elt_data *elt_data;
6b4827ad 1865 struct hist_field *key_field;
02205a67 1866 unsigned int i, n_str;
6b4827ad 1867
af6a29bc
TZ		 1868 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1869 if (!elt_data)
1870 return -ENOMEM;
1871
6b4827ad
TZ		 1872 for_each_hist_key_field(i, hist_data) {
1873 key_field = hist_data->fields[i];
1874
1875 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 1876 elt_data->comm = kzalloc(size, GFP_KERNEL);
1877 if (!elt_data->comm) {
1878 kfree(elt_data);
6b4827ad 1879 return -ENOMEM;
af6a29bc 1880 }
6b4827ad
TZ		 1881 break;
1882 }
1883 }
1884
50450603 1885 n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
02205a67
TZ		 1886
1887 size = STR_VAR_LEN_MAX;
1888
1889 for (i = 0; i < n_str; i++) {
1890 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1891 if (!elt_data->field_var_str[i]) {
1892 hist_elt_data_free(elt_data);
1893 return -ENOMEM;
1894 }
1895 }
1896
af6a29bc
TZ		 1897 elt->private_data = elt_data;
1898
6b4827ad
TZ		 1899 return 0;
1900}
1901
af6a29bc 1902static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
6b4827ad 1903{
af6a29bc 1904 struct hist_elt_data *elt_data = elt->private_data;
6b4827ad 1905
af6a29bc
TZ		 1906 if (elt_data->comm)
1907 save_comm(elt_data->comm, current);
6b4827ad
TZ		 1908}
1909
af6a29bc
TZ		 1910static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1911 .elt_alloc = hist_trigger_elt_data_alloc,
1912 .elt_free = hist_trigger_elt_data_free,
1913 .elt_init = hist_trigger_elt_data_init,
6b4827ad
TZ		 1914};
1915
2ece94fb
TZ		 1916static const char *get_hist_field_flags(struct hist_field *hist_field)
1917{
1918 const char *flags_str = NULL;
1919
1920 if (hist_field->flags & HIST_FIELD_FL_HEX)
1921 flags_str = "hex";
1922 else if (hist_field->flags & HIST_FIELD_FL_SYM)
1923 flags_str = "sym";
1924 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1925 flags_str = "sym-offset";
1926 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1927 flags_str = "execname";
1928 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1929 flags_str = "syscall";
1930 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1931 flags_str = "log2";
1932 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1933 flags_str = "usecs";
1934
1935 return flags_str;
1936}
1937
100719dc
TZ		 1938static void expr_field_str(struct hist_field *field, char *expr)
1939{
067fe038
TZ		 1940 if (field->flags & HIST_FIELD_FL_VAR_REF)
1941 strcat(expr, "$");
1942
100719dc
TZ		 1943 strcat(expr, hist_field_name(field, 0));
1944
1945 if (field->flags) {
1946 const char *flags_str = get_hist_field_flags(field);
1947
1948 if (flags_str) {
1949 strcat(expr, ".");
1950 strcat(expr, flags_str);
1951 }
1952 }
1953}
1954
1955static char *expr_str(struct hist_field *field, unsigned int level)
1956{
1957 char *expr;
1958
1959 if (level > 1)
1960 return NULL;
1961
1962 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1963 if (!expr)
1964 return NULL;
1965
1966 if (!field->operands[0]) {
1967 expr_field_str(field, expr);
1968 return expr;
1969 }
1970
1971 if (field->operator == FIELD_OP_UNARY_MINUS) {
1972 char *subexpr;
1973
1974 strcat(expr, "-(");
1975 subexpr = expr_str(field->operands[0], ++level);
1976 if (!subexpr) {
1977 kfree(expr);
1978 return NULL;
1979 }
1980 strcat(expr, subexpr);
1981 strcat(expr, ")");
1982
1983 kfree(subexpr);
1984
1985 return expr;
1986 }
1987
1988 expr_field_str(field->operands[0], expr);
1989
1990 switch (field->operator) {
1991 case FIELD_OP_MINUS:
1992 strcat(expr, "-");
1993 break;
1994 case FIELD_OP_PLUS:
1995 strcat(expr, "+");
1996 break;
1997 default:
1998 kfree(expr);
1999 return NULL;
2000 }
2001
2002 expr_field_str(field->operands[1], expr);
2003
2004 return expr;
2005}
2006
2007static int contains_operator(char *str)
2008{
2009 enum field_op_id field_op = FIELD_OP_NONE;
2010 char *op;
2011
2012 op = strpbrk(str, "+-");
2013 if (!op)
2014 return FIELD_OP_NONE;
2015
2016 switch (*op) {
2017 case '-':
2018 if (*str == '-')
2019 field_op = FIELD_OP_UNARY_MINUS;
2020 else
2021 field_op = FIELD_OP_MINUS;
2022 break;
2023 case '+':
2024 field_op = FIELD_OP_PLUS;
2025 break;
2026 default:
2027 break;
2028 }
2029
2030 return field_op;
2031}
2032
5819eadd
TZ		 2033static void destroy_hist_field(struct hist_field *hist_field,
2034 unsigned int level)
7ef224d1 2035{
5819eadd
TZ		 2036 unsigned int i;
2037
100719dc 2038 if (level > 3)
5819eadd
TZ		 2039 return;
2040
2041 if (!hist_field)
2042 return;
2043
2044 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2045 destroy_hist_field(hist_field->operands[i], level + 1);
2046
30350d65 2047 kfree(hist_field->var.name);
100719dc 2048 kfree(hist_field->name);
19a9facd 2049 kfree(hist_field->type);
30350d65 2050
7ef224d1
TZ		 2051 kfree(hist_field);
2052}
2053
b559d003
TZ		 2054static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2055 struct ftrace_event_field *field,
30350d65
TZ		 2056 unsigned long flags,
2057 char *var_name)
7ef224d1
TZ		 2058{
2059 struct hist_field *hist_field;
2060
2061 if (field && is_function_field(field))
2062 return NULL;
2063
2064 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2065 if (!hist_field)
2066 return NULL;
2067
b559d003
TZ		 2068 hist_field->hist_data = hist_data;
2069
100719dc
TZ		 2070 if (flags & HIST_FIELD_FL_EXPR)
2071 goto out; /* caller will populate */
2072
067fe038
TZ		 2073 if (flags & HIST_FIELD_FL_VAR_REF) {
2074 hist_field->fn = hist_field_var_ref;
2075 goto out;
2076 }
2077
7ef224d1
TZ		 2078 if (flags & HIST_FIELD_FL_HITCOUNT) {
2079 hist_field->fn = hist_field_counter;
19a9facd
TZ		 2080 hist_field->size = sizeof(u64);
2081 hist_field->type = kstrdup("u64", GFP_KERNEL);
2082 if (!hist_field->type)
2083 goto free;
7ef224d1
TZ		 2084 goto out;
2085 }
2086
69a0200c
TZ		 2087 if (flags & HIST_FIELD_FL_STACKTRACE) {
2088 hist_field->fn = hist_field_none;
2089 goto out;
2090 }
2091
4b94f5b7 2092 if (flags & HIST_FIELD_FL_LOG2) {
5819eadd 2093 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
4b94f5b7 2094 hist_field->fn = hist_field_log2;
30350d65 2095 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
5819eadd 2096 hist_field->size = hist_field->operands[0]->size;
19a9facd
TZ		 2097 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2098 if (!hist_field->type)
2099 goto free;
4b94f5b7
NK		 2100 goto out;
2101 }
2102
ad42febe
TZ		 2103 if (flags & HIST_FIELD_FL_TIMESTAMP) {
2104 hist_field->fn = hist_field_timestamp;
2105 hist_field->size = sizeof(u64);
19a9facd
TZ		 2106 hist_field->type = kstrdup("u64", GFP_KERNEL);
2107 if (!hist_field->type)
2108 goto free;
ad42febe
TZ		 2109 goto out;
2110 }
2111
432480c5
TZ		 2112 if (WARN_ON_ONCE(!field))
2113 goto out;
2114
7ef224d1
TZ		 2115 if (is_string_field(field)) {
2116 flags |= HIST_FIELD_FL_STRING;
79e577cb 2117
19a9facd
TZ		 2118 hist_field->size = MAX_FILTER_STR_VAL;
2119 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2120 if (!hist_field->type)
2121 goto free;
2122
79e577cb
NK		 2123 if (field->filter_type == FILTER_STATIC_STRING)
2124 hist_field->fn = hist_field_string;
2125 else if (field->filter_type == FILTER_DYN_STRING)
2126 hist_field->fn = hist_field_dynstring;
2127 else
2128 hist_field->fn = hist_field_pstring;
7ef224d1 2129 } else {
19a9facd
TZ		 2130 hist_field->size = field->size;
2131 hist_field->is_signed = field->is_signed;
2132 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2133 if (!hist_field->type)
2134 goto free;
2135
7ef224d1
TZ		 2136 hist_field->fn = select_value_fn(field->size,
2137 field->is_signed);
2138 if (!hist_field->fn) {
5819eadd 2139 destroy_hist_field(hist_field, 0);
7ef224d1
TZ		 2140 return NULL;
2141 }
2142 }
2143 out:
2144 hist_field->field = field;
2145 hist_field->flags = flags;
2146
30350d65
TZ		 2147 if (var_name) {
2148 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2149 if (!hist_field->var.name)
2150 goto free;
2151 }
2152
7ef224d1 2153 return hist_field;
30350d65
TZ		 2154 free:
2155 destroy_hist_field(hist_field, 0);
2156 return NULL;
7ef224d1
TZ		 2157}
2158
2159static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2160{
2161 unsigned int i;
2162
30350d65 2163 for (i = 0; i < HIST_FIELDS_MAX; i++) {
7ef224d1 2164 if (hist_data->fields[i]) {
5819eadd 2165 destroy_hist_field(hist_data->fields[i], 0);
7ef224d1
TZ		 2166 hist_data->fields[i] = NULL;
2167 }
2168 }
2169}
2170
067fe038
TZ		 2171static int init_var_ref(struct hist_field *ref_field,
2172 struct hist_field *var_field,
2173 char *system, char *event_name)
2174{
2175 int err = 0;
2176
2177 ref_field->var.idx = var_field->var.idx;
2178 ref_field->var.hist_data = var_field->hist_data;
2179 ref_field->size = var_field->size;
2180 ref_field->is_signed = var_field->is_signed;
2181 ref_field->flags |= var_field->flags &
2182 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2183
2184 if (system) {
2185 ref_field->system = kstrdup(system, GFP_KERNEL);
2186 if (!ref_field->system)
2187 return -ENOMEM;
2188 }
2189
2190 if (event_name) {
2191 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2192 if (!ref_field->event_name) {
2193 err = -ENOMEM;
2194 goto free;
2195 }
2196 }
2197
2198 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2199 if (!ref_field->name) {
2200 err = -ENOMEM;
2201 goto free;
2202 }
2203
2204 ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2205 if (!ref_field->type) {
2206 err = -ENOMEM;
2207 goto free;
2208 }
2209 out:
2210 return err;
2211 free:
2212 kfree(ref_field->system);
2213 kfree(ref_field->event_name);
2214 kfree(ref_field->name);
2215
2216 goto out;
2217}
2218
2219static struct hist_field *create_var_ref(struct hist_field *var_field,
2220 char *system, char *event_name)
2221{
2222 unsigned long flags = HIST_FIELD_FL_VAR_REF;
2223 struct hist_field *ref_field;
2224
2225 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2226 if (ref_field) {
2227 if (init_var_ref(ref_field, var_field, system, event_name)) {
2228 destroy_hist_field(ref_field, 0);
2229 return NULL;
2230 }
2231 }
2232
2233 return ref_field;
2234}
2235
2236static bool is_var_ref(char *var_name)
2237{
2238 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2239 return false;
2240
2241 return true;
2242}
2243
2244static char *field_name_from_var(struct hist_trigger_data *hist_data,
2245 char *var_name)
2246{
2247 char *name, *field;
2248 unsigned int i;
2249
2250 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2251 name = hist_data->attrs->var_defs.name[i];
2252
2253 if (strcmp(var_name, name) == 0) {
2254 field = hist_data->attrs->var_defs.expr[i];
2255 if (contains_operator(field) || is_var_ref(field))
2256 continue;
2257 return field;
2258 }
2259 }
2260
2261 return NULL;
2262}
2263
2264static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2265 char *system, char *event_name,
2266 char *var_name)
2267{
2268 struct trace_event_call *call;
2269
2270 if (system && event_name) {
2271 call = hist_data->event_file->event_call;
2272
2273 if (strcmp(system, call->class->system) != 0)
2274 return NULL;
2275
2276 if (strcmp(event_name, trace_event_name(call)) != 0)
2277 return NULL;
2278 }
2279
2280 if (!!system != !!event_name)
2281 return NULL;
2282
2283 if (!is_var_ref(var_name))
2284 return NULL;
2285
2286 var_name++;
2287
2288 return field_name_from_var(hist_data, var_name);
2289}
2290
2291static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2292 char *system, char *event_name,
2293 char *var_name)
2294{
2295 struct hist_field *var_field = NULL, *ref_field = NULL;
2296
2297 if (!is_var_ref(var_name))
2298 return NULL;
2299
2300 var_name++;
2301
2302 var_field = find_event_var(hist_data, system, event_name, var_name);
2303 if (var_field)
2304 ref_field = create_var_ref(var_field, system, event_name);
2305
2306 return ref_field;
2307}
2308
100719dc
TZ		 2309static struct ftrace_event_field *
2310parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2311 char *field_str, unsigned long *flags)
2312{
2313 struct ftrace_event_field *field = NULL;
2314 char *field_name, *modifier, *str;
2315
2316 modifier = str = kstrdup(field_str, GFP_KERNEL);
2317 if (!modifier)
2318 return ERR_PTR(-ENOMEM);
2319
2320 field_name = strsep(&modifier, ".");
2321 if (modifier) {
2322 if (strcmp(modifier, "hex") == 0)
2323 *flags |= HIST_FIELD_FL_HEX;
2324 else if (strcmp(modifier, "sym") == 0)
2325 *flags |= HIST_FIELD_FL_SYM;
2326 else if (strcmp(modifier, "sym-offset") == 0)
2327 *flags |= HIST_FIELD_FL_SYM_OFFSET;
2328 else if ((strcmp(modifier, "execname") == 0) &&
2329 (strcmp(field_name, "common_pid") == 0))
2330 *flags |= HIST_FIELD_FL_EXECNAME;
2331 else if (strcmp(modifier, "syscall") == 0)
2332 *flags |= HIST_FIELD_FL_SYSCALL;
2333 else if (strcmp(modifier, "log2") == 0)
2334 *flags |= HIST_FIELD_FL_LOG2;
2335 else if (strcmp(modifier, "usecs") == 0)
2336 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2337 else {
2338 field = ERR_PTR(-EINVAL);
2339 goto out;
2340 }
2341 }
2342
2343 if (strcmp(field_name, "common_timestamp") == 0) {
2344 *flags |= HIST_FIELD_FL_TIMESTAMP;
2345 hist_data->enable_timestamps = true;
2346 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2347 hist_data->attrs->ts_in_usecs = true;
2348 } else {
2349 field = trace_find_event_field(file->event_call, field_name);
2350 if (!field || !field->size) {
2351 field = ERR_PTR(-EINVAL);
2352 goto out;
2353 }
2354 }
2355 out:
2356 kfree(str);
2357
2358 return field;
2359}
2360
2361static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2362 struct trace_event_file *file, char *str,
2363 unsigned long *flags, char *var_name)
2364{
067fe038 2365 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
100719dc
TZ		 2366 struct ftrace_event_field *field = NULL;
2367 struct hist_field *hist_field = NULL;
2368 int ret = 0;
2369
067fe038
TZ		 2370 s = strchr(str, '.');
2371 if (s) {
2372 s = strchr(++s, '.');
2373 if (s) {
2374 ref_system = strsep(&str, ".");
2375 if (!str) {
2376 ret = -EINVAL;
2377 goto out;
2378 }
2379 ref_event = strsep(&str, ".");
2380 if (!str) {
2381 ret = -EINVAL;
2382 goto out;
2383 }
2384 ref_var = str;
2385 }
2386 }
2387
2388 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2389 if (!s) {
2390 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2391 if (hist_field) {
2392 hist_data->var_refs[hist_data->n_var_refs] = hist_field;
2393 hist_field->var_ref_idx = hist_data->n_var_refs++;
2394 return hist_field;
2395 }
2396 } else
2397 str = s;
2398
100719dc
TZ		 2399 field = parse_field(hist_data, file, str, flags);
2400 if (IS_ERR(field)) {
2401 ret = PTR_ERR(field);
2402 goto out;
2403 }
2404
2405 hist_field = create_hist_field(hist_data, field, *flags, var_name);
2406 if (!hist_field) {
2407 ret = -ENOMEM;
2408 goto out;
2409 }
2410
2411 return hist_field;
2412 out:
2413 return ERR_PTR(ret);
2414}
2415
2416static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2417 struct trace_event_file *file,
2418 char *str, unsigned long flags,
2419 char *var_name, unsigned int level);
2420
2421static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2422 struct trace_event_file *file,
2423 char *str, unsigned long flags,
2424 char *var_name, unsigned int level)
2425{
2426 struct hist_field *operand1, *expr = NULL;
2427 unsigned long operand_flags;
2428 int ret = 0;
2429 char *s;
2430
2431 /* we support only -(xxx) i.e. explicit parens required */
2432
2433 if (level > 3) {
2434 ret = -EINVAL;
2435 goto free;
2436 }
2437
2438 str++; /* skip leading '-' */
2439
2440 s = strchr(str, '(');
2441 if (s)
2442 str++;
2443 else {
2444 ret = -EINVAL;
2445 goto free;
2446 }
2447
2448 s = strrchr(str, ')');
2449 if (s)
2450 *s = '\0';
2451 else {
2452 ret = -EINVAL; /* no closing ')' */
2453 goto free;
2454 }
2455
2456 flags |= HIST_FIELD_FL_EXPR;
2457 expr = create_hist_field(hist_data, NULL, flags, var_name);
2458 if (!expr) {
2459 ret = -ENOMEM;
2460 goto free;
2461 }
2462
2463 operand_flags = 0;
2464 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2465 if (IS_ERR(operand1)) {
2466 ret = PTR_ERR(operand1);
2467 goto free;
2468 }
2469
2470 expr->flags |= operand1->flags &
2471 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2472 expr->fn = hist_field_unary_minus;
2473 expr->operands[0] = operand1;
2474 expr->operator = FIELD_OP_UNARY_MINUS;
2475 expr->name = expr_str(expr, 0);
19a9facd
TZ		 2476 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2477 if (!expr->type) {
2478 ret = -ENOMEM;
2479 goto free;
2480 }
100719dc
TZ		 2481
2482 return expr;
2483 free:
2484 destroy_hist_field(expr, 0);
2485 return ERR_PTR(ret);
2486}
2487
2488static int check_expr_operands(struct hist_field *operand1,
2489 struct hist_field *operand2)
2490{
2491 unsigned long operand1_flags = operand1->flags;
2492 unsigned long operand2_flags = operand2->flags;
2493
2494 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2495 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS))
2496 return -EINVAL;
2497
2498 return 0;
2499}
2500
2501static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2502 struct trace_event_file *file,
2503 char *str, unsigned long flags,
2504 char *var_name, unsigned int level)
2505{
2506 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2507 unsigned long operand_flags;
2508 int field_op, ret = -EINVAL;
2509 char *sep, *operand1_str;
2510
2511 if (level > 3)
2512 return ERR_PTR(-EINVAL);
2513
2514 field_op = contains_operator(str);
2515
2516 if (field_op == FIELD_OP_NONE)
2517 return parse_atom(hist_data, file, str, &flags, var_name);
2518
2519 if (field_op == FIELD_OP_UNARY_MINUS)
2520 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2521
2522 switch (field_op) {
2523 case FIELD_OP_MINUS:
2524 sep = "-";
2525 break;
2526 case FIELD_OP_PLUS:
2527 sep = "+";
2528 break;
2529 default:
2530 goto free;
2531 }
2532
2533 operand1_str = strsep(&str, sep);
2534 if (!operand1_str || !str)
2535 goto free;
2536
2537 operand_flags = 0;
2538 operand1 = parse_atom(hist_data, file, operand1_str,
2539 &operand_flags, NULL);
2540 if (IS_ERR(operand1)) {
2541 ret = PTR_ERR(operand1);
2542 operand1 = NULL;
2543 goto free;
2544 }
2545
2546 /* rest of string could be another expression e.g. b+c in a+b+c */
2547 operand_flags = 0;
2548 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2549 if (IS_ERR(operand2)) {
2550 ret = PTR_ERR(operand2);
2551 operand2 = NULL;
2552 goto free;
2553 }
2554
2555 ret = check_expr_operands(operand1, operand2);
2556 if (ret)
2557 goto free;
2558
2559 flags |= HIST_FIELD_FL_EXPR;
2560
2561 flags |= operand1->flags &
2562 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2563
2564 expr = create_hist_field(hist_data, NULL, flags, var_name);
2565 if (!expr) {
2566 ret = -ENOMEM;
2567 goto free;
2568 }
2569
067fe038
TZ		 2570 operand1->read_once = true;
2571 operand2->read_once = true;
2572
100719dc
TZ		 2573 expr->operands[0] = operand1;
2574 expr->operands[1] = operand2;
2575 expr->operator = field_op;
2576 expr->name = expr_str(expr, 0);
19a9facd
TZ		 2577 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2578 if (!expr->type) {
2579 ret = -ENOMEM;
2580 goto free;
2581 }
100719dc
TZ		 2582
2583 switch (field_op) {
2584 case FIELD_OP_MINUS:
2585 expr->fn = hist_field_minus;
2586 break;
2587 case FIELD_OP_PLUS:
2588 expr->fn = hist_field_plus;
2589 break;
2590 default:
2591 goto free;
2592 }
2593
2594 return expr;
2595 free:
2596 destroy_hist_field(operand1, 0);
2597 destroy_hist_field(operand2, 0);
2598 destroy_hist_field(expr, 0);
2599
2600 return ERR_PTR(ret);
2601}
2602
02205a67
TZ		 2603static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2604 struct trace_event_file *file)
2605{
2606 struct event_trigger_data *test;
2607
2608 list_for_each_entry_rcu(test, &file->triggers, list) {
2609 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2610 if (test->private_data == hist_data)
2611 return test->filter_str;
2612 }
2613 }
2614
2615 return NULL;
2616}
2617
2618static struct event_command trigger_hist_cmd;
2619static int event_hist_trigger_func(struct event_command *cmd_ops,
2620 struct trace_event_file *file,
2621 char *glob, char *cmd, char *param);
2622
2623static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2624 struct hist_trigger_data *hist_data,
2625 unsigned int n_keys)
2626{
2627 struct hist_field *target_hist_field, *hist_field;
2628 unsigned int n, i, j;
2629
2630 if (hist_data->n_fields - hist_data->n_vals != n_keys)
2631 return false;
2632
2633 i = hist_data->n_vals;
2634 j = target_hist_data->n_vals;
2635
2636 for (n = 0; n < n_keys; n++) {
2637 hist_field = hist_data->fields[i + n];
2638 target_hist_field = target_hist_data->fields[j + n];
2639
2640 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2641 return false;
2642 if (hist_field->size != target_hist_field->size)
2643 return false;
2644 if (hist_field->is_signed != target_hist_field->is_signed)
2645 return false;
2646 }
2647
2648 return true;
2649}
2650
2651static struct hist_trigger_data *
2652find_compatible_hist(struct hist_trigger_data *target_hist_data,
2653 struct trace_event_file *file)
2654{
2655 struct hist_trigger_data *hist_data;
2656 struct event_trigger_data *test;
2657 unsigned int n_keys;
2658
2659 n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2660
2661 list_for_each_entry_rcu(test, &file->triggers, list) {
2662 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2663 hist_data = test->private_data;
2664
2665 if (compatible_keys(target_hist_data, hist_data, n_keys))
2666 return hist_data;
2667 }
2668 }
2669
2670 return NULL;
2671}
2672
2673static struct trace_event_file *event_file(struct trace_array *tr,
2674 char *system, char *event_name)
2675{
2676 struct trace_event_file *file;
2677
2678 file = find_event_file(tr, system, event_name);
2679 if (!file)
2680 return ERR_PTR(-EINVAL);
2681
2682 return file;
2683}
2684
2685static struct hist_field *
2686find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2687 char *system, char *event_name, char *field_name)
2688{
2689 struct hist_field *event_var;
2690 char *synthetic_name;
2691
2692 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2693 if (!synthetic_name)
2694 return ERR_PTR(-ENOMEM);
2695
2696 strcpy(synthetic_name, "synthetic_");
2697 strcat(synthetic_name, field_name);
2698
2699 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2700
2701 kfree(synthetic_name);
2702
2703 return event_var;
2704}
2705
2706/**
2707 * create_field_var_hist - Automatically create a histogram and var for a field
2708 * @target_hist_data: The target hist trigger
2709 * @subsys_name: Optional subsystem name
2710 * @event_name: Optional event name
2711 * @field_name: The name of the field (and the resulting variable)
2712 *
2713 * Hist trigger actions fetch data from variables, not directly from
2714 * events. However, for convenience, users are allowed to directly
2715 * specify an event field in an action, which will be automatically
2716 * converted into a variable on their behalf.
2717
2718 * If a user specifies a field on an event that isn't the event the
2719 * histogram currently being defined (the target event histogram), the
2720 * only way that can be accomplished is if a new hist trigger is
2721 * created and the field variable defined on that.
2722 *
2723 * This function creates a new histogram compatible with the target
2724 * event (meaning a histogram with the same key as the target
2725 * histogram), and creates a variable for the specified field, but
2726 * with 'synthetic_' prepended to the variable name in order to avoid
2727 * collision with normal field variables.
2728 *
2729 * Return: The variable created for the field.
2730 */
c282a386 2731static struct hist_field *
02205a67
TZ		 2732create_field_var_hist(struct hist_trigger_data *target_hist_data,
2733 char *subsys_name, char *event_name, char *field_name)
2734{
2735 struct trace_array *tr = target_hist_data->event_file->tr;
2736 struct hist_field *event_var = ERR_PTR(-EINVAL);
2737 struct hist_trigger_data *hist_data;
2738 unsigned int i, n, first = true;
2739 struct field_var_hist *var_hist;
2740 struct trace_event_file *file;
2741 struct hist_field *key_field;
2742 char *saved_filter;
2743 char *cmd;
2744 int ret;
2745
2746 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX)
2747 return ERR_PTR(-EINVAL);
2748
2749 file = event_file(tr, subsys_name, event_name);
2750
2751 if (IS_ERR(file)) {
2752 ret = PTR_ERR(file);
2753 return ERR_PTR(ret);
2754 }
2755
2756 /*
2757 * Look for a histogram compatible with target. We'll use the
2758 * found histogram specification to create a new matching
2759 * histogram with our variable on it. target_hist_data is not
2760 * yet a registered histogram so we can't use that.
2761 */
2762 hist_data = find_compatible_hist(target_hist_data, file);
2763 if (!hist_data)
2764 return ERR_PTR(-EINVAL);
2765
2766 /* See if a synthetic field variable has already been created */
2767 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2768 event_name, field_name);
2769 if (!IS_ERR_OR_NULL(event_var))
2770 return event_var;
2771
2772 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2773 if (!var_hist)
2774 return ERR_PTR(-ENOMEM);
2775
2776 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2777 if (!cmd) {
2778 kfree(var_hist);
2779 return ERR_PTR(-ENOMEM);
2780 }
2781
2782 /* Use the same keys as the compatible histogram */
2783 strcat(cmd, "keys=");
2784
2785 for_each_hist_key_field(i, hist_data) {
2786 key_field = hist_data->fields[i];
2787 if (!first)
2788 strcat(cmd, ",");
2789 strcat(cmd, key_field->field->name);
2790 first = false;
2791 }
2792
2793 /* Create the synthetic field variable specification */
2794 strcat(cmd, ":synthetic_");
2795 strcat(cmd, field_name);
2796 strcat(cmd, "=");
2797 strcat(cmd, field_name);
2798
2799 /* Use the same filter as the compatible histogram */
2800 saved_filter = find_trigger_filter(hist_data, file);
2801 if (saved_filter) {
2802 strcat(cmd, " if ");
2803 strcat(cmd, saved_filter);
2804 }
2805
2806 var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2807 if (!var_hist->cmd) {
2808 kfree(cmd);
2809 kfree(var_hist);
2810 return ERR_PTR(-ENOMEM);
2811 }
2812
2813 /* Save the compatible histogram information */
2814 var_hist->hist_data = hist_data;
2815
2816 /* Create the new histogram with our variable */
2817 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2818 "", "hist", cmd);
2819 if (ret) {
2820 kfree(cmd);
2821 kfree(var_hist->cmd);
2822 kfree(var_hist);
2823 return ERR_PTR(ret);
2824 }
2825
2826 kfree(cmd);
2827
2828 /* If we can't find the variable, something went wrong */
2829 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2830 event_name, field_name);
2831 if (IS_ERR_OR_NULL(event_var)) {
2832 kfree(var_hist->cmd);
2833 kfree(var_hist);
2834 return ERR_PTR(-EINVAL);
2835 }
2836
2837 n = target_hist_data->n_field_var_hists;
2838 target_hist_data->field_var_hists[n] = var_hist;
2839 target_hist_data->n_field_var_hists++;
2840
2841 return event_var;
2842}
2843
c282a386 2844static struct hist_field *
02205a67
TZ		 2845find_target_event_var(struct hist_trigger_data *hist_data,
2846 char *subsys_name, char *event_name, char *var_name)
2847{
2848 struct trace_event_file *file = hist_data->event_file;
2849 struct hist_field *hist_field = NULL;
2850
2851 if (subsys_name) {
2852 struct trace_event_call *call;
2853
2854 if (!event_name)
2855 return NULL;
2856
2857 call = file->event_call;
2858
2859 if (strcmp(subsys_name, call->class->system) != 0)
2860 return NULL;
2861
2862 if (strcmp(event_name, trace_event_name(call)) != 0)
2863 return NULL;
2864 }
2865
2866 hist_field = find_var_field(hist_data, var_name);
2867
2868 return hist_field;
2869}
2870
2871static inline void __update_field_vars(struct tracing_map_elt *elt,
2872 struct ring_buffer_event *rbe,
2873 void *rec,
2874 struct field_var **field_vars,
2875 unsigned int n_field_vars,
2876 unsigned int field_var_str_start)
2877{
2878 struct hist_elt_data *elt_data = elt->private_data;
2879 unsigned int i, j, var_idx;
2880 u64 var_val;
2881
2882 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2883 struct field_var *field_var = field_vars[i];
2884 struct hist_field *var = field_var->var;
2885 struct hist_field *val = field_var->val;
2886
2887 var_val = val->fn(val, elt, rbe, rec);
2888 var_idx = var->var.idx;
2889
2890 if (val->flags & HIST_FIELD_FL_STRING) {
2891 char *str = elt_data->field_var_str[j++];
2892 char *val_str = (char *)(uintptr_t)var_val;
2893
2894 strncpy(str, val_str, STR_VAR_LEN_MAX);
2895 var_val = (u64)(uintptr_t)str;
2896 }
2897 tracing_map_set_var(elt, var_idx, var_val);
2898 }
2899}
2900
2901static void update_field_vars(struct hist_trigger_data *hist_data,
2902 struct tracing_map_elt *elt,
2903 struct ring_buffer_event *rbe,
2904 void *rec)
2905{
2906 __update_field_vars(elt, rbe, rec, hist_data->field_vars,
2907 hist_data->n_field_vars, 0);
2908}
2909
50450603
TZ		 2910static void update_max_vars(struct hist_trigger_data *hist_data,
2911 struct tracing_map_elt *elt,
2912 struct ring_buffer_event *rbe,
2913 void *rec)
2914{
2915 __update_field_vars(elt, rbe, rec, hist_data->max_vars,
2916 hist_data->n_max_vars, hist_data->n_field_var_str);
2917}
2918
02205a67
TZ		 2919static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2920 struct trace_event_file *file,
2921 char *name, int size, const char *type)
2922{
2923 struct hist_field *var;
2924 int idx;
2925
2926 if (find_var(hist_data, file, name) && !hist_data->remove) {
2927 var = ERR_PTR(-EINVAL);
2928 goto out;
2929 }
2930
2931 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2932 if (!var) {
2933 var = ERR_PTR(-ENOMEM);
2934 goto out;
2935 }
2936
2937 idx = tracing_map_add_var(hist_data->map);
2938 if (idx < 0) {
2939 kfree(var);
2940 var = ERR_PTR(-EINVAL);
2941 goto out;
2942 }
2943
2944 var->flags = HIST_FIELD_FL_VAR;
2945 var->var.idx = idx;
2946 var->var.hist_data = var->hist_data = hist_data;
2947 var->size = size;
2948 var->var.name = kstrdup(name, GFP_KERNEL);
2949 var->type = kstrdup(type, GFP_KERNEL);
2950 if (!var->var.name || !var->type) {
2951 kfree(var->var.name);
2952 kfree(var->type);
2953 kfree(var);
2954 var = ERR_PTR(-ENOMEM);
2955 }
2956 out:
2957 return var;
2958}
2959
2960static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2961 struct trace_event_file *file,
2962 char *field_name)
2963{
2964 struct hist_field *val = NULL, *var = NULL;
2965 unsigned long flags = HIST_FIELD_FL_VAR;
2966 struct field_var *field_var;
2967 int ret = 0;
2968
2969 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2970 ret = -EINVAL;
2971 goto err;
2972 }
2973
2974 val = parse_atom(hist_data, file, field_name, &flags, NULL);
2975 if (IS_ERR(val)) {
2976 ret = PTR_ERR(val);
2977 goto err;
2978 }
2979
2980 var = create_var(hist_data, file, field_name, val->size, val->type);
2981 if (IS_ERR(var)) {
2982 kfree(val);
2983 ret = PTR_ERR(var);
2984 goto err;
2985 }
2986
2987 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2988 if (!field_var) {
2989 kfree(val);
2990 kfree(var);
2991 ret = -ENOMEM;
2992 goto err;
2993 }
2994
2995 field_var->var = var;
2996 field_var->val = val;
2997 out:
2998 return field_var;
2999 err:
3000 field_var = ERR_PTR(ret);
3001 goto out;
3002}
3003
3004/**
3005 * create_target_field_var - Automatically create a variable for a field
3006 * @target_hist_data: The target hist trigger
3007 * @subsys_name: Optional subsystem name
3008 * @event_name: Optional event name
3009 * @var_name: The name of the field (and the resulting variable)
3010 *
3011 * Hist trigger actions fetch data from variables, not directly from
3012 * events. However, for convenience, users are allowed to directly
3013 * specify an event field in an action, which will be automatically
3014 * converted into a variable on their behalf.
3015
3016 * This function creates a field variable with the name var_name on
3017 * the hist trigger currently being defined on the target event. If
3018 * subsys_name and event_name are specified, this function simply
3019 * verifies that they do in fact match the target event subsystem and
3020 * event name.
3021 *
3022 * Return: The variable created for the field.
3023 */
c282a386 3024static struct field_var *
02205a67
TZ		 3025create_target_field_var(struct hist_trigger_data *target_hist_data,
3026 char *subsys_name, char *event_name, char *var_name)
3027{
3028 struct trace_event_file *file = target_hist_data->event_file;
3029
3030 if (subsys_name) {
3031 struct trace_event_call *call;
3032
3033 if (!event_name)
3034 return NULL;
3035
3036 call = file->event_call;
3037
3038 if (strcmp(subsys_name, call->class->system) != 0)
3039 return NULL;
3040
3041 if (strcmp(event_name, trace_event_name(call)) != 0)
3042 return NULL;
3043 }
3044
3045 return create_field_var(target_hist_data, file, var_name);
3046}
3047
50450603
TZ		 3048static void onmax_print(struct seq_file *m,
3049 struct hist_trigger_data *hist_data,
3050 struct tracing_map_elt *elt,
3051 struct action_data *data)
3052{
3053 unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx;
3054
3055 seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx));
3056
3057 for (i = 0; i < hist_data->n_max_vars; i++) {
3058 struct hist_field *save_val = hist_data->max_vars[i]->val;
3059 struct hist_field *save_var = hist_data->max_vars[i]->var;
3060 u64 val;
3061
3062 save_var_idx = save_var->var.idx;
3063
3064 val = tracing_map_read_var(elt, save_var_idx);
3065
3066 if (save_val->flags & HIST_FIELD_FL_STRING) {
3067 seq_printf(m, " %s: %-32s", save_var->var.name,
3068 (char *)(uintptr_t)(val));
3069 } else
3070 seq_printf(m, " %s: %10llu", save_var->var.name, val);
3071 }
3072}
3073
3074static void onmax_save(struct hist_trigger_data *hist_data,
3075 struct tracing_map_elt *elt, void *rec,
3076 struct ring_buffer_event *rbe,
3077 struct action_data *data, u64 *var_ref_vals)
3078{
3079 unsigned int max_idx = data->onmax.max_var->var.idx;
3080 unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx;
3081
3082 u64 var_val, max_val;
3083
3084 var_val = var_ref_vals[max_var_ref_idx];
3085 max_val = tracing_map_read_var(elt, max_idx);
3086
3087 if (var_val <= max_val)
3088 return;
3089
3090 tracing_map_set_var(elt, max_idx, var_val);
3091
3092 update_max_vars(hist_data, elt, rbe, rec);
3093}
3094
3095static void onmax_destroy(struct action_data *data)
3096{
3097 unsigned int i;
3098
3099 destroy_hist_field(data->onmax.max_var, 0);
3100 destroy_hist_field(data->onmax.var, 0);
3101
3102 kfree(data->onmax.var_str);
3103 kfree(data->onmax.fn_name);
3104
3105 for (i = 0; i < data->n_params; i++)
3106 kfree(data->params[i]);
3107
3108 kfree(data);
3109}
3110
3111static int onmax_create(struct hist_trigger_data *hist_data,
3112 struct action_data *data)
3113{
3114 struct trace_event_file *file = hist_data->event_file;
3115 struct hist_field *var_field, *ref_field, *max_var;
3116 unsigned int var_ref_idx = hist_data->n_var_refs;
3117 struct field_var *field_var;
3118 char *onmax_var_str, *param;
3119 unsigned long flags;
3120 unsigned int i;
3121 int ret = 0;
3122
3123 onmax_var_str = data->onmax.var_str;
3124 if (onmax_var_str[0] != '$')
3125 return -EINVAL;
3126 onmax_var_str++;
3127
3128 var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
3129 if (!var_field)
3130 return -EINVAL;
3131
3132 flags = HIST_FIELD_FL_VAR_REF;
3133 ref_field = create_hist_field(hist_data, NULL, flags, NULL);
3134 if (!ref_field)
3135 return -ENOMEM;
3136
3137 if (init_var_ref(ref_field, var_field, NULL, NULL)) {
3138 destroy_hist_field(ref_field, 0);
3139 ret = -ENOMEM;
3140 goto out;
3141 }
3142 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3143 ref_field->var_ref_idx = hist_data->n_var_refs++;
3144 data->onmax.var = ref_field;
3145
3146 data->fn = onmax_save;
3147 data->onmax.max_var_ref_idx = var_ref_idx;
3148 max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
3149 if (IS_ERR(max_var)) {
3150 ret = PTR_ERR(max_var);
3151 goto out;
3152 }
3153 data->onmax.max_var = max_var;
3154
3155 for (i = 0; i < data->n_params; i++) {
3156 param = kstrdup(data->params[i], GFP_KERNEL);
3157 if (!param) {
3158 ret = -ENOMEM;
3159 goto out;
3160 }
3161
3162 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3163 if (IS_ERR(field_var)) {
3164 ret = PTR_ERR(field_var);
3165 kfree(param);
3166 goto out;
3167 }
3168
3169 hist_data->max_vars[hist_data->n_max_vars++] = field_var;
3170 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3171 hist_data->n_max_var_str++;
3172
3173 kfree(param);
3174 }
3175 out:
3176 return ret;
3177}
3178
3179static int parse_action_params(char *params, struct action_data *data)
3180{
3181 char *param, *saved_param;
3182 int ret = 0;
3183
3184 while (params) {
3185 if (data->n_params >= SYNTH_FIELDS_MAX)
3186 goto out;
3187
3188 param = strsep(&params, ",");
3189 if (!param) {
3190 ret = -EINVAL;
3191 goto out;
3192 }
3193
3194 param = strstrip(param);
3195 if (strlen(param) < 2) {
3196 ret = -EINVAL;
3197 goto out;
3198 }
3199
3200 saved_param = kstrdup(param, GFP_KERNEL);
3201 if (!saved_param) {
3202 ret = -ENOMEM;
3203 goto out;
3204 }
3205
3206 data->params[data->n_params++] = saved_param;
3207 }
3208 out:
3209 return ret;
3210}
3211
3212static struct action_data *onmax_parse(char *str)
3213{
3214 char *onmax_fn_name, *onmax_var_str;
3215 struct action_data *data;
3216 int ret = -EINVAL;
3217
3218 data = kzalloc(sizeof(*data), GFP_KERNEL);
3219 if (!data)
3220 return ERR_PTR(-ENOMEM);
3221
3222 onmax_var_str = strsep(&str, ")");
3223 if (!onmax_var_str || !str) {
3224 ret = -EINVAL;
3225 goto free;
3226 }
3227
3228 data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL);
3229 if (!data->onmax.var_str) {
3230 ret = -ENOMEM;
3231 goto free;
3232 }
3233
3234 strsep(&str, ".");
3235 if (!str)
3236 goto free;
3237
3238 onmax_fn_name = strsep(&str, "(");
3239 if (!onmax_fn_name || !str)
3240 goto free;
3241
3242 if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) {
3243 char *params = strsep(&str, ")");
3244
3245 if (!params) {
3246 ret = -EINVAL;
3247 goto free;
3248 }
3249
3250 ret = parse_action_params(params, data);
3251 if (ret)
3252 goto free;
3253 } else
3254 goto free;
3255
3256 data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL);
3257 if (!data->onmax.fn_name) {
3258 ret = -ENOMEM;
3259 goto free;
3260 }
3261 out:
3262 return data;
3263 free:
3264 onmax_destroy(data);
3265 data = ERR_PTR(ret);
3266 goto out;
3267}
3268
c282a386
TZ		 3269static void onmatch_destroy(struct action_data *data)
3270{
3271 unsigned int i;
3272
3273 mutex_lock(&synth_event_mutex);
3274
3275 kfree(data->onmatch.match_event);
3276 kfree(data->onmatch.match_event_system);
3277 kfree(data->onmatch.synth_event_name);
3278
3279 for (i = 0; i < data->n_params; i++)
3280 kfree(data->params[i]);
3281
3282 if (data->onmatch.synth_event)
3283 data->onmatch.synth_event->ref--;
3284
3285 kfree(data);
3286
3287 mutex_unlock(&synth_event_mutex);
3288}
3289
02205a67
TZ		 3290static void destroy_field_var(struct field_var *field_var)
3291{
3292 if (!field_var)
3293 return;
3294
3295 destroy_hist_field(field_var->var, 0);
3296 destroy_hist_field(field_var->val, 0);
3297
3298 kfree(field_var);
3299}
3300
3301static void destroy_field_vars(struct hist_trigger_data *hist_data)
3302{
3303 unsigned int i;
3304
3305 for (i = 0; i < hist_data->n_field_vars; i++)
3306 destroy_field_var(hist_data->field_vars[i]);
3307}
3308
c282a386
TZ		 3309static void save_field_var(struct hist_trigger_data *hist_data,
3310 struct field_var *field_var)
02205a67
TZ		 3311{
3312 hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3313
3314 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3315 hist_data->n_field_var_str++;
3316}
3317
c282a386
TZ		 3318
3319static void destroy_synth_var_refs(struct hist_trigger_data *hist_data)
3320{
3321 unsigned int i;
3322
3323 for (i = 0; i < hist_data->n_synth_var_refs; i++)
3324 destroy_hist_field(hist_data->synth_var_refs[i], 0);
3325}
3326
3327static void save_synth_var_ref(struct hist_trigger_data *hist_data,
3328 struct hist_field *var_ref)
3329{
3330 hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref;
3331
3332 hist_data->var_refs[hist_data->n_var_refs] = var_ref;
3333 var_ref->var_ref_idx = hist_data->n_var_refs++;
3334}
3335
3336static int check_synth_field(struct synth_event *event,
3337 struct hist_field *hist_field,
3338 unsigned int field_pos)
3339{
3340 struct synth_field *field;
3341
3342 if (field_pos >= event->n_fields)
3343 return -EINVAL;
3344
3345 field = event->fields[field_pos];
3346
3347 if (strcmp(field->type, hist_field->type) != 0)
3348 return -EINVAL;
3349
3350 return 0;
3351}
3352
c282a386
TZ		 3353static struct hist_field *
3354onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data,
3355 char *system, char *event, char *var)
3356{
3357 struct hist_field *hist_field;
3358
3359 var++; /* skip '$' */
3360
3361 hist_field = find_target_event_var(hist_data, system, event, var);
3362 if (!hist_field) {
3363 if (!system) {
3364 system = data->onmatch.match_event_system;
3365 event = data->onmatch.match_event;
3366 }
3367
3368 hist_field = find_event_var(hist_data, system, event, var);
3369 }
3370
3371 return hist_field;
3372}
3373
3374static struct hist_field *
3375onmatch_create_field_var(struct hist_trigger_data *hist_data,
3376 struct action_data *data, char *system,
3377 char *event, char *var)
3378{
3379 struct hist_field *hist_field = NULL;
3380 struct field_var *field_var;
3381
3382 /*
3383 * First try to create a field var on the target event (the
3384 * currently being defined). This will create a variable for
3385 * unqualified fields on the target event, or if qualified,
3386 * target fields that have qualified names matching the target.
3387 */
3388 field_var = create_target_field_var(hist_data, system, event, var);
3389
3390 if (field_var && !IS_ERR(field_var)) {
3391 save_field_var(hist_data, field_var);
3392 hist_field = field_var->var;
3393 } else {
3394 field_var = NULL;
3395 /*
3396 * If no explicit system.event is specfied, default to
3397 * looking for fields on the onmatch(system.event.xxx)
3398 * event.
3399 */
3400 if (!system) {
3401 system = data->onmatch.match_event_system;
3402 event = data->onmatch.match_event;
3403 }
3404
3405 /*
3406 * At this point, we're looking at a field on another
3407 * event. Because we can't modify a hist trigger on
3408 * another event to add a variable for a field, we need
3409 * to create a new trigger on that event and create the
3410 * variable at the same time.
3411 */
3412 hist_field = create_field_var_hist(hist_data, system, event, var);
3413 if (IS_ERR(hist_field))
3414 goto free;
3415 }
3416 out:
3417 return hist_field;
3418 free:
3419 destroy_field_var(field_var);
3420 hist_field = NULL;
3421 goto out;
3422}
3423
3424static int onmatch_create(struct hist_trigger_data *hist_data,
3425 struct trace_event_file *file,
3426 struct action_data *data)
3427{
3428 char *event_name, *param, *system = NULL;
3429 struct hist_field *hist_field, *var_ref;
3430 unsigned int i, var_ref_idx;
3431 unsigned int field_pos = 0;
3432 struct synth_event *event;
3433 int ret = 0;
3434
3435 mutex_lock(&synth_event_mutex);
3436 event = find_synth_event(data->onmatch.synth_event_name);
3437 if (!event) {
3438 mutex_unlock(&synth_event_mutex);
3439 return -EINVAL;
3440 }
3441 event->ref++;
3442 mutex_unlock(&synth_event_mutex);
3443
3444 var_ref_idx = hist_data->n_var_refs;
3445
3446 for (i = 0; i < data->n_params; i++) {
3447 char *p;
3448
3449 p = param = kstrdup(data->params[i], GFP_KERNEL);
3450 if (!param) {
3451 ret = -ENOMEM;
3452 goto err;
3453 }
3454
3455 system = strsep(&param, ".");
3456 if (!param) {
3457 param = (char *)system;
3458 system = event_name = NULL;
3459 } else {
3460 event_name = strsep(&param, ".");
3461 if (!param) {
3462 kfree(p);
3463 ret = -EINVAL;
3464 goto err;
3465 }
3466 }
3467
3468 if (param[0] == '$')
3469 hist_field = onmatch_find_var(hist_data, data, system,
3470 event_name, param);
3471 else
3472 hist_field = onmatch_create_field_var(hist_data, data,
3473 system,
3474 event_name,
3475 param);
3476
3477 if (!hist_field) {
3478 kfree(p);
3479 ret = -EINVAL;
3480 goto err;
3481 }
3482
3483 if (check_synth_field(event, hist_field, field_pos) == 0) {
3484 var_ref = create_var_ref(hist_field, system, event_name);
3485 if (!var_ref) {
3486 kfree(p);
3487 ret = -ENOMEM;
3488 goto err;
3489 }
3490
3491 save_synth_var_ref(hist_data, var_ref);
3492 field_pos++;
3493 kfree(p);
3494 continue;
3495 }
3496
3497 kfree(p);
3498 ret = -EINVAL;
3499 goto err;
3500 }
3501
3502 if (field_pos != event->n_fields) {
3503 ret = -EINVAL;
3504 goto err;
3505 }
3506
3507 data->fn = action_trace;
3508 data->onmatch.synth_event = event;
3509 data->onmatch.var_ref_idx = var_ref_idx;
3510 out:
3511 return ret;
3512 err:
3513 mutex_lock(&synth_event_mutex);
3514 event->ref--;
3515 mutex_unlock(&synth_event_mutex);
3516
3517 goto out;
3518}
3519
3520static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3521{
3522 char *match_event, *match_event_system;
3523 char *synth_event_name, *params;
3524 struct action_data *data;
3525 int ret = -EINVAL;
3526
3527 data = kzalloc(sizeof(*data), GFP_KERNEL);
3528 if (!data)
3529 return ERR_PTR(-ENOMEM);
3530
3531 match_event = strsep(&str, ")");
3532 if (!match_event || !str)
3533 goto free;
3534
3535 match_event_system = strsep(&match_event, ".");
3536 if (!match_event)
3537 goto free;
3538
3539 if (IS_ERR(event_file(tr, match_event_system, match_event)))
3540 goto free;
3541
3542 data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
3543 if (!data->onmatch.match_event) {
3544 ret = -ENOMEM;
3545 goto free;
3546 }
3547
3548 data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL);
3549 if (!data->onmatch.match_event_system) {
3550 ret = -ENOMEM;
3551 goto free;
3552 }
3553
3554 strsep(&str, ".");
3555 if (!str)
3556 goto free;
3557
3558 synth_event_name = strsep(&str, "(");
3559 if (!synth_event_name || !str)
3560 goto free;
3561
3562 data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
3563 if (!data->onmatch.synth_event_name) {
3564 ret = -ENOMEM;
3565 goto free;
3566 }
3567
3568 params = strsep(&str, ")");
3569 if (!params || !str || (str && strlen(str)))
3570 goto free;
3571
3572 ret = parse_action_params(params, data);
3573 if (ret)
3574 goto free;
3575 out:
3576 return data;
3577 free:
3578 onmatch_destroy(data);
3579 data = ERR_PTR(ret);
3580 goto out;
3581}
3582
7ef224d1
TZ		 3583static int create_hitcount_val(struct hist_trigger_data *hist_data)
3584{
3585 hist_data->fields[HITCOUNT_IDX] =
30350d65 3586 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
7ef224d1
TZ		 3587 if (!hist_data->fields[HITCOUNT_IDX])
3588 return -ENOMEM;
3589
3590 hist_data->n_vals++;
30350d65 3591 hist_data->n_fields++;
7ef224d1
TZ		 3592
3593 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3594 return -EINVAL;
3595
3596 return 0;
3597}
3598
30350d65
TZ		 3599static int __create_val_field(struct hist_trigger_data *hist_data,
3600 unsigned int val_idx,
3601 struct trace_event_file *file,
3602 char *var_name, char *field_str,
3603 unsigned long flags)
f2606835 3604{
100719dc 3605 struct hist_field *hist_field;
f2606835
TZ		 3606 int ret = 0;
3607
100719dc
TZ		 3608 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3609 if (IS_ERR(hist_field)) {
3610 ret = PTR_ERR(hist_field);
f2606835
TZ		 3611 goto out;
3612 }
3613
100719dc
TZ		 3614 hist_data->fields[val_idx] = hist_field;
3615
f2606835 3616 ++hist_data->n_vals;
30350d65 3617 ++hist_data->n_fields;
f2606835 3618
30350d65 3619 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
f2606835
TZ		 3620 ret = -EINVAL;
3621 out:
3622 return ret;
3623}
3624
30350d65
TZ		 3625static int create_val_field(struct hist_trigger_data *hist_data,
3626 unsigned int val_idx,
3627 struct trace_event_file *file,
3628 char *field_str)
3629{
3630 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3631 return -EINVAL;
3632
3633 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3634}
3635
3636static int create_var_field(struct hist_trigger_data *hist_data,
3637 unsigned int val_idx,
3638 struct trace_event_file *file,
3639 char *var_name, char *expr_str)
3640{
3641 unsigned long flags = 0;
3642
3643 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3644 return -EINVAL;
3645 if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3646 return -EINVAL;
3647 }
3648
3649 flags |= HIST_FIELD_FL_VAR;
3650 hist_data->n_vars++;
3651 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3652 return -EINVAL;
3653
3654 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3655}
3656
7ef224d1
TZ		 3657static int create_val_fields(struct hist_trigger_data *hist_data,
3658 struct trace_event_file *file)
3659{
f2606835 3660 char *fields_str, *field_str;
30350d65 3661 unsigned int i, j = 1;
7ef224d1
TZ		 3662 int ret;
3663
3664 ret = create_hitcount_val(hist_data);
f2606835
TZ		 3665 if (ret)
3666 goto out;
7ef224d1 3667
f2606835
TZ		 3668 fields_str = hist_data->attrs->vals_str;
3669 if (!fields_str)
3670 goto out;
3671
3672 strsep(&fields_str, "=");
3673 if (!fields_str)
3674 goto out;
3675
3676 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3677 j < TRACING_MAP_VALS_MAX; i++) {
3678 field_str = strsep(&fields_str, ",");
3679 if (!field_str)
3680 break;
30350d65 3681
f2606835
TZ		 3682 if (strcmp(field_str, "hitcount") == 0)
3683 continue;
30350d65 3684
f2606835
TZ		 3685 ret = create_val_field(hist_data, j++, file, field_str);
3686 if (ret)
3687 goto out;
3688 }
30350d65 3689
f2606835
TZ		 3690 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3691 ret = -EINVAL;
3692 out:
7ef224d1
TZ		 3693 return ret;
3694}
3695
3696static int create_key_field(struct hist_trigger_data *hist_data,
3697 unsigned int key_idx,
76a3b0c8 3698 unsigned int key_offset,
7ef224d1
TZ		 3699 struct trace_event_file *file,
3700 char *field_str)
3701{
30350d65 3702 struct hist_field *hist_field = NULL;
100719dc 3703
7ef224d1
TZ		 3704 unsigned long flags = 0;
3705 unsigned int key_size;
3706 int ret = 0;
3707
30350d65 3708 if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
7ef224d1
TZ		 3709 return -EINVAL;
3710
3711 flags |= HIST_FIELD_FL_KEY;
3712
69a0200c
TZ		 3713 if (strcmp(field_str, "stacktrace") == 0) {
3714 flags |= HIST_FIELD_FL_STACKTRACE;
3715 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
30350d65 3716 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
69a0200c 3717 } else {
100719dc
TZ		 3718 hist_field = parse_expr(hist_data, file, field_str, flags,
3719 NULL, 0);
3720 if (IS_ERR(hist_field)) {
3721 ret = PTR_ERR(hist_field);
3722 goto out;
69a0200c
TZ		 3723 }
3724
067fe038
TZ		 3725 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
3726 destroy_hist_field(hist_field, 0);
3727 ret = -EINVAL;
3728 goto out;
3729 }
3730
100719dc 3731 key_size = hist_field->size;
7ef224d1
TZ		 3732 }
3733
100719dc 3734 hist_data->fields[key_idx] = hist_field;
7ef224d1
TZ		 3735
3736 key_size = ALIGN(key_size, sizeof(u64));
3737 hist_data->fields[key_idx]->size = key_size;
76a3b0c8 3738 hist_data->fields[key_idx]->offset = key_offset;
100719dc 3739
76a3b0c8 3740 hist_data->key_size += key_size;
100719dc 3741
7ef224d1
TZ		 3742 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3743 ret = -EINVAL;
3744 goto out;
3745 }
3746
3747 hist_data->n_keys++;
30350d65 3748 hist_data->n_fields++;
7ef224d1
TZ		 3749
3750 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3751 return -EINVAL;
3752
3753 ret = key_size;
3754 out:
3755 return ret;
3756}
3757
3758static int create_key_fields(struct hist_trigger_data *hist_data,
3759 struct trace_event_file *file)
3760{
76a3b0c8 3761 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
7ef224d1
TZ		 3762 char *fields_str, *field_str;
3763 int ret = -EINVAL;
3764
3765 fields_str = hist_data->attrs->keys_str;
3766 if (!fields_str)
3767 goto out;
3768
3769 strsep(&fields_str, "=");
3770 if (!fields_str)
3771 goto out;
3772
76a3b0c8 3773 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
7ef224d1
TZ		 3774 field_str = strsep(&fields_str, ",");
3775 if (!field_str)
3776 break;
76a3b0c8
TZ		 3777 ret = create_key_field(hist_data, i, key_offset,
3778 file, field_str);
7ef224d1
TZ		 3779 if (ret < 0)
3780 goto out;
76a3b0c8 3781 key_offset += ret;
7ef224d1
TZ		 3782 }
3783 if (fields_str) {
3784 ret = -EINVAL;
3785 goto out;
3786 }
3787 ret = 0;
3788 out:
3789 return ret;
3790}
3791
30350d65
TZ		 3792static int create_var_fields(struct hist_trigger_data *hist_data,
3793 struct trace_event_file *file)
3794{
3795 unsigned int i, j = hist_data->n_vals;
3796 int ret = 0;
3797
3798 unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3799
3800 for (i = 0; i < n_vars; i++) {
3801 char *var_name = hist_data->attrs->var_defs.name[i];
3802 char *expr = hist_data->attrs->var_defs.expr[i];
3803
3804 ret = create_var_field(hist_data, j++, file, var_name, expr);
3805 if (ret)
3806 goto out;
3807 }
3808 out:
3809 return ret;
3810}
3811
3812static void free_var_defs(struct hist_trigger_data *hist_data)
3813{
3814 unsigned int i;
3815
3816 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3817 kfree(hist_data->attrs->var_defs.name[i]);
3818 kfree(hist_data->attrs->var_defs.expr[i]);
3819 }
3820
3821 hist_data->attrs->var_defs.n_vars = 0;
3822}
3823
3824static int parse_var_defs(struct hist_trigger_data *hist_data)
3825{
3826 char *s, *str, *var_name, *field_str;
3827 unsigned int i, j, n_vars = 0;
3828 int ret = 0;
3829
3830 for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3831 str = hist_data->attrs->assignment_str[i];
3832 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3833 field_str = strsep(&str, ",");
3834 if (!field_str)
3835 break;
3836
3837 var_name = strsep(&field_str, "=");
3838 if (!var_name || !field_str) {
3839 ret = -EINVAL;
3840 goto free;
3841 }
3842
3843 if (n_vars == TRACING_MAP_VARS_MAX) {
3844 ret = -EINVAL;
3845 goto free;
3846 }
3847
3848 s = kstrdup(var_name, GFP_KERNEL);
3849 if (!s) {
3850 ret = -ENOMEM;
3851 goto free;
3852 }
3853 hist_data->attrs->var_defs.name[n_vars] = s;
3854
3855 s = kstrdup(field_str, GFP_KERNEL);
3856 if (!s) {
3857 kfree(hist_data->attrs->var_defs.name[n_vars]);
3858 ret = -ENOMEM;
3859 goto free;
3860 }
3861 hist_data->attrs->var_defs.expr[n_vars++] = s;
3862
3863 hist_data->attrs->var_defs.n_vars = n_vars;
3864 }
3865 }
3866
3867 return ret;
3868 free:
3869 free_var_defs(hist_data);
3870
3871 return ret;
3872}
3873
7ef224d1
TZ		 3874static int create_hist_fields(struct hist_trigger_data *hist_data,
3875 struct trace_event_file *file)
3876{
3877 int ret;
3878
30350d65
TZ		 3879 ret = parse_var_defs(hist_data);
3880 if (ret)
3881 goto out;
3882
7ef224d1
TZ		 3883 ret = create_val_fields(hist_data, file);
3884 if (ret)
3885 goto out;
3886
30350d65 3887 ret = create_var_fields(hist_data, file);
7ef224d1
TZ		 3888 if (ret)
3889 goto out;
3890
30350d65
TZ		 3891 ret = create_key_fields(hist_data, file);
3892 if (ret)
3893 goto out;
7ef224d1 3894 out:
30350d65
TZ		 3895 free_var_defs(hist_data);
3896
7ef224d1
TZ		 3897 return ret;
3898}
3899
e62347d2
TZ		 3900static int is_descending(const char *str)
3901{
3902 if (!str)
3903 return 0;
3904
3905 if (strcmp(str, "descending") == 0)
3906 return 1;
3907
3908 if (strcmp(str, "ascending") == 0)
3909 return 0;
3910
3911 return -EINVAL;
3912}
3913
7ef224d1
TZ		 3914static int create_sort_keys(struct hist_trigger_data *hist_data)
3915{
e62347d2 3916 char *fields_str = hist_data->attrs->sort_key_str;
e62347d2
TZ		 3917 struct tracing_map_sort_key *sort_key;
3918 int descending, ret = 0;
30350d65 3919 unsigned int i, j, k;
e62347d2
TZ		 3920
3921 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3922
3923 if (!fields_str)
3924 goto out;
3925
3926 strsep(&fields_str, "=");
3927 if (!fields_str) {
3928 ret = -EINVAL;
3929 goto out;
3930 }
3931
3932 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
85013256 3933 struct hist_field *hist_field;
e62347d2 3934 char *field_str, *field_name;
85013256 3935 const char *test_name;
e62347d2
TZ		 3936
3937 sort_key = &hist_data->sort_keys[i];
3938
3939 field_str = strsep(&fields_str, ",");
3940 if (!field_str) {
3941 if (i == 0)
3942 ret = -EINVAL;
3943 break;
3944 }
3945
3946 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
3947 ret = -EINVAL;
3948 break;
3949 }
7ef224d1 3950
e62347d2
TZ		 3951 field_name = strsep(&field_str, ".");
3952 if (!field_name) {
3953 ret = -EINVAL;
3954 break;
3955 }
3956
3957 if (strcmp(field_name, "hitcount") == 0) {
3958 descending = is_descending(field_str);
3959 if (descending < 0) {
3960 ret = descending;
3961 break;
3962 }
3963 sort_key->descending = descending;
3964 continue;
3965 }
7ef224d1 3966
30350d65
TZ		 3967 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
3968 unsigned int idx;
3969
85013256 3970 hist_field = hist_data->fields[j];
30350d65
TZ		 3971 if (hist_field->flags & HIST_FIELD_FL_VAR)
3972 continue;
3973
3974 idx = k++;
3975
85013256
TZ		 3976 test_name = hist_field_name(hist_field, 0);
3977
3978 if (strcmp(field_name, test_name) == 0) {
30350d65 3979 sort_key->field_idx = idx;
e62347d2
TZ		 3980 descending = is_descending(field_str);
3981 if (descending < 0) {
3982 ret = descending;
3983 goto out;
3984 }
3985 sort_key->descending = descending;
3986 break;
3987 }
3988 }
3989 if (j == hist_data->n_fields) {
3990 ret = -EINVAL;
3991 break;
3992 }
3993 }
30350d65 3994
e62347d2
TZ		 3995 hist_data->n_sort_keys = i;
3996 out:
7ef224d1
TZ		 3997 return ret;
3998}
3999
0212e2aa
TZ		 4000static void destroy_actions(struct hist_trigger_data *hist_data)
4001{
4002 unsigned int i;
4003
4004 for (i = 0; i < hist_data->n_actions; i++) {
4005 struct action_data *data = hist_data->actions[i];
4006
c282a386
TZ		 4007 if (data->fn == action_trace)
4008 onmatch_destroy(data);
50450603
TZ		 4009 else if (data->fn == onmax_save)
4010 onmax_destroy(data);
c282a386
TZ		 4011 else
4012 kfree(data);
0212e2aa
TZ		 4013 }
4014}
4015
4016static int parse_actions(struct hist_trigger_data *hist_data)
4017{
c282a386
TZ		 4018 struct trace_array *tr = hist_data->event_file->tr;
4019 struct action_data *data;
0212e2aa
TZ		 4020 unsigned int i;
4021 int ret = 0;
4022 char *str;
4023
4024 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4025 str = hist_data->attrs->action_str[i];
c282a386
TZ		 4026
4027 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) {
4028 char *action_str = str + strlen("onmatch(");
4029
4030 data = onmatch_parse(tr, action_str);
4031 if (IS_ERR(data)) {
4032 ret = PTR_ERR(data);
4033 break;
4034 }
4035 data->fn = action_trace;
50450603
TZ		 4036 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) {
4037 char *action_str = str + strlen("onmax(");
4038
4039 data = onmax_parse(action_str);
4040 if (IS_ERR(data)) {
4041 ret = PTR_ERR(data);
4042 break;
4043 }
4044 data->fn = onmax_save;
c282a386
TZ		 4045 } else {
4046 ret = -EINVAL;
4047 break;
4048 }
4049
4050 hist_data->actions[hist_data->n_actions++] = data;
0212e2aa
TZ		 4051 }
4052
4053 return ret;
4054}
4055
4056static int create_actions(struct hist_trigger_data *hist_data,
4057 struct trace_event_file *file)
4058{
4059 struct action_data *data;
4060 unsigned int i;
4061 int ret = 0;
4062
4063 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4064 data = hist_data->actions[i];
c282a386
TZ		 4065
4066 if (data->fn == action_trace) {
4067 ret = onmatch_create(hist_data, file, data);
4068 if (ret)
4069 return ret;
50450603
TZ		 4070 } else if (data->fn == onmax_save) {
4071 ret = onmax_create(hist_data, data);
4072 if (ret)
4073 return ret;
c282a386 4074 }
0212e2aa
TZ		 4075 }
4076
4077 return ret;
4078}
4079
50450603
TZ		 4080static void print_actions(struct seq_file *m,
4081 struct hist_trigger_data *hist_data,
4082 struct tracing_map_elt *elt)
4083{
4084 unsigned int i;
4085
4086 for (i = 0; i < hist_data->n_actions; i++) {
4087 struct action_data *data = hist_data->actions[i];
4088
4089 if (data->fn == onmax_save)
4090 onmax_print(m, hist_data, elt, data);
4091 }
4092}
4093
4094static void print_onmax_spec(struct seq_file *m,
4095 struct hist_trigger_data *hist_data,
4096 struct action_data *data)
4097{
4098 unsigned int i;
4099
4100 seq_puts(m, ":onmax(");
4101 seq_printf(m, "%s", data->onmax.var_str);
4102 seq_printf(m, ").%s(", data->onmax.fn_name);
4103
4104 for (i = 0; i < hist_data->n_max_vars; i++) {
4105 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name);
4106 if (i < hist_data->n_max_vars - 1)
4107 seq_puts(m, ",");
4108 }
4109 seq_puts(m, ")");
4110}
4111
c282a386
TZ		 4112static void print_onmatch_spec(struct seq_file *m,
4113 struct hist_trigger_data *hist_data,
4114 struct action_data *data)
4115{
4116 unsigned int i;
4117
4118 seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system,
4119 data->onmatch.match_event);
4120
4121 seq_printf(m, "%s(", data->onmatch.synth_event->name);
4122
4123 for (i = 0; i < data->n_params; i++) {
4124 if (i)
4125 seq_puts(m, ",");
4126 seq_printf(m, "%s", data->params[i]);
4127 }
4128
4129 seq_puts(m, ")");
4130}
4131
4132static void print_actions_spec(struct seq_file *m,
4133 struct hist_trigger_data *hist_data)
4134{
4135 unsigned int i;
4136
4137 for (i = 0; i < hist_data->n_actions; i++) {
4138 struct action_data *data = hist_data->actions[i];
4139
4140 if (data->fn == action_trace)
4141 print_onmatch_spec(m, hist_data, data);
50450603
TZ		 4142 else if (data->fn == onmax_save)
4143 print_onmax_spec(m, hist_data, data);
c282a386
TZ		 4144 }
4145}
4146
02205a67
TZ		 4147static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4148{
4149 unsigned int i;
4150
4151 for (i = 0; i < hist_data->n_field_var_hists; i++) {
4152 kfree(hist_data->field_var_hists[i]->cmd);
4153 kfree(hist_data->field_var_hists[i]);
4154 }
4155}
4156
7ef224d1
TZ		 4157static void destroy_hist_data(struct hist_trigger_data *hist_data)
4158{
0212e2aa
TZ		 4159 if (!hist_data)
4160 return;
4161
7ef224d1
TZ		 4162 destroy_hist_trigger_attrs(hist_data->attrs);
4163 destroy_hist_fields(hist_data);
4164 tracing_map_destroy(hist_data->map);
0212e2aa
TZ		 4165
4166 destroy_actions(hist_data);
02205a67
TZ		 4167 destroy_field_vars(hist_data);
4168 destroy_field_var_hists(hist_data);
c282a386 4169 destroy_synth_var_refs(hist_data);
0212e2aa 4170
7ef224d1
TZ		 4171 kfree(hist_data);
4172}
4173
4174static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4175{
4176 struct tracing_map *map = hist_data->map;
4177 struct ftrace_event_field *field;
4178 struct hist_field *hist_field;
d50c744e 4179 int i, idx;
7ef224d1
TZ		 4180
4181 for_each_hist_field(i, hist_data) {
4182 hist_field = hist_data->fields[i];
4183 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4184 tracing_map_cmp_fn_t cmp_fn;
4185
4186 field = hist_field->field;
4187
69a0200c
TZ		 4188 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4189 cmp_fn = tracing_map_cmp_none;
ad42febe
TZ		 4190 else if (!field)
4191 cmp_fn = tracing_map_cmp_num(hist_field->size,
4192 hist_field->is_signed);
69a0200c 4193 else if (is_string_field(field))
7ef224d1
TZ		 4194 cmp_fn = tracing_map_cmp_string;
4195 else
4196 cmp_fn = tracing_map_cmp_num(field->size,
4197 field->is_signed);
76a3b0c8
TZ		 4198 idx = tracing_map_add_key_field(map,
4199 hist_field->offset,
4200 cmp_fn);
30350d65 4201 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
7ef224d1
TZ		 4202 idx = tracing_map_add_sum_field(map);
4203
4204 if (idx < 0)
4205 return idx;
30350d65
TZ		 4206
4207 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4208 idx = tracing_map_add_var(map);
4209 if (idx < 0)
4210 return idx;
4211 hist_field->var.idx = idx;
4212 hist_field->var.hist_data = hist_data;
4213 }
7ef224d1
TZ		 4214 }
4215
4216 return 0;
4217}
4218
4219static struct hist_trigger_data *
4220create_hist_data(unsigned int map_bits,
4221 struct hist_trigger_attrs *attrs,
30350d65
TZ		 4222 struct trace_event_file *file,
4223 bool remove)
7ef224d1 4224{
6b4827ad 4225 const struct tracing_map_ops *map_ops = NULL;
7ef224d1
TZ		 4226 struct hist_trigger_data *hist_data;
4227 int ret = 0;
4228
4229 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4230 if (!hist_data)
4231 return ERR_PTR(-ENOMEM);
4232
4233 hist_data->attrs = attrs;
30350d65 4234 hist_data->remove = remove;
067fe038 4235 hist_data->event_file = file;
7ef224d1 4236
0212e2aa
TZ		 4237 ret = parse_actions(hist_data);
4238 if (ret)
4239 goto free;
4240
7ef224d1
TZ		 4241 ret = create_hist_fields(hist_data, file);
4242 if (ret)
4243 goto free;
4244
4245 ret = create_sort_keys(hist_data);
4246 if (ret)
4247 goto free;
4248
af6a29bc 4249 map_ops = &hist_trigger_elt_data_ops;
6b4827ad 4250
7ef224d1 4251 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6b4827ad 4252 map_ops, hist_data);
7ef224d1
TZ		 4253 if (IS_ERR(hist_data->map)) {
4254 ret = PTR_ERR(hist_data->map);
4255 hist_data->map = NULL;
4256 goto free;
4257 }
4258
4259 ret = create_tracing_map_fields(hist_data);
4260 if (ret)
4261 goto free;
7ef224d1
TZ		 4262 out:
4263 return hist_data;
4264 free:
4265 hist_data->attrs = NULL;
4266
4267 destroy_hist_data(hist_data);
4268
4269 hist_data = ERR_PTR(ret);
4270
4271 goto out;
4272}
4273
4274static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
fbd302cb 4275 struct tracing_map_elt *elt, void *rec,
067fe038
TZ		 4276 struct ring_buffer_event *rbe,
4277 u64 *var_ref_vals)
7ef224d1 4278{
067fe038 4279 struct hist_elt_data *elt_data;
7ef224d1 4280 struct hist_field *hist_field;
30350d65 4281 unsigned int i, var_idx;
7ef224d1
TZ		 4282 u64 hist_val;
4283
067fe038
TZ		 4284 elt_data = elt->private_data;
4285 elt_data->var_ref_vals = var_ref_vals;
4286
7ef224d1
TZ		 4287 for_each_hist_val_field(i, hist_data) {
4288 hist_field = hist_data->fields[i];
df35d93b 4289 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 4290 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4291 var_idx = hist_field->var.idx;
4292 tracing_map_set_var(elt, var_idx, hist_val);
4293 continue;
4294 }
7ef224d1
TZ		 4295 tracing_map_update_sum(elt, i, hist_val);
4296 }
30350d65
TZ		 4297
4298 for_each_hist_key_field(i, hist_data) {
4299 hist_field = hist_data->fields[i];
4300 if (hist_field->flags & HIST_FIELD_FL_VAR) {
df35d93b 4301 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 4302 var_idx = hist_field->var.idx;
4303 tracing_map_set_var(elt, var_idx, hist_val);
4304 }
4305 }
02205a67
TZ		 4306
4307 update_field_vars(hist_data, elt, rbe, rec);
7ef224d1
TZ		 4308}
4309
6a475cb1
TZ		 4310static inline void add_to_key(char *compound_key, void *key,
4311 struct hist_field *key_field, void *rec)
4312{
4313 size_t size = key_field->size;
4314
4315 if (key_field->flags & HIST_FIELD_FL_STRING) {
4316 struct ftrace_event_field *field;
4317
4318 field = key_field->field;
4319 if (field->filter_type == FILTER_DYN_STRING)
4320 size = *(u32 *)(rec + field->offset) >> 16;
4321 else if (field->filter_type == FILTER_PTR_STRING)
4322 size = strlen(key);
4323 else if (field->filter_type == FILTER_STATIC_STRING)
4324 size = field->size;
4325
4326 /* ensure NULL-termination */
4327 if (size > key_field->size - 1)
4328 size = key_field->size - 1;
4329 }
4330
4331 memcpy(compound_key + key_field->offset, key, size);
4332}
4333
0212e2aa
TZ		 4334static void
4335hist_trigger_actions(struct hist_trigger_data *hist_data,
4336 struct tracing_map_elt *elt, void *rec,
4337 struct ring_buffer_event *rbe, u64 *var_ref_vals)
4338{
4339 struct action_data *data;
4340 unsigned int i;
4341
4342 for (i = 0; i < hist_data->n_actions; i++) {
4343 data = hist_data->actions[i];
4344 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals);
4345 }
4346}
4347
1ac4f51c 4348static void event_hist_trigger(struct event_trigger_data *data, void *rec,
fbd302cb 4349 struct ring_buffer_event *rbe)
7ef224d1
TZ		 4350{
4351 struct hist_trigger_data *hist_data = data->private_data;
6a475cb1 4352 bool use_compound_key = (hist_data->n_keys > 1);
69a0200c 4353 unsigned long entries[HIST_STACKTRACE_DEPTH];
067fe038 4354 u64 var_ref_vals[TRACING_MAP_VARS_MAX];
76a3b0c8 4355 char compound_key[HIST_KEY_SIZE_MAX];
df35d93b 4356 struct tracing_map_elt *elt = NULL;
69a0200c 4357 struct stack_trace stacktrace;
7ef224d1 4358 struct hist_field *key_field;
7ef224d1
TZ		 4359 u64 field_contents;
4360 void *key = NULL;
4361 unsigned int i;
4362
6a475cb1 4363 memset(compound_key, 0, hist_data->key_size);
76a3b0c8 4364
7ef224d1
TZ		 4365 for_each_hist_key_field(i, hist_data) {
4366 key_field = hist_data->fields[i];
4367
69a0200c
TZ		 4368 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4369 stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
4370 stacktrace.entries = entries;
4371 stacktrace.nr_entries = 0;
4372 stacktrace.skip = HIST_STACKTRACE_SKIP;
76a3b0c8 4373
69a0200c
TZ		 4374 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
4375 save_stack_trace(&stacktrace);
4376
4377 key = entries;
4378 } else {
df35d93b 4379 field_contents = key_field->fn(key_field, elt, rbe, rec);
6a475cb1 4380 if (key_field->flags & HIST_FIELD_FL_STRING) {
69a0200c 4381 key = (void *)(unsigned long)field_contents;
6a475cb1
TZ		 4382 use_compound_key = true;
4383 } else
69a0200c 4384 key = (void *)&field_contents;
76a3b0c8 4385 }
6a475cb1
TZ		 4386
4387 if (use_compound_key)
4388 add_to_key(compound_key, key, key_field, rec);
7ef224d1
TZ		 4389 }
4390
6a475cb1 4391 if (use_compound_key)
76a3b0c8
TZ		 4392 key = compound_key;
4393
067fe038
TZ		 4394 if (hist_data->n_var_refs &&
4395 !resolve_var_refs(hist_data, key, var_ref_vals, false))
4396 return;
4397
7ef224d1 4398 elt = tracing_map_insert(hist_data->map, key);
067fe038
TZ		 4399 if (!elt)
4400 return;
4401
4402 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
0212e2aa
TZ		 4403
4404 if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4405 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals);
7ef224d1
TZ		 4406}
4407
69a0200c
TZ		 4408static void hist_trigger_stacktrace_print(struct seq_file *m,
4409 unsigned long *stacktrace_entries,
4410 unsigned int max_entries)
4411{
4412 char str[KSYM_SYMBOL_LEN];
4413 unsigned int spaces = 8;
4414 unsigned int i;
4415
4416 for (i = 0; i < max_entries; i++) {
4417 if (stacktrace_entries[i] == ULONG_MAX)
4418 return;
4419
4420 seq_printf(m, "%*c", 1 + spaces, ' ');
4421 sprint_symbol(str, stacktrace_entries[i]);
4422 seq_printf(m, "%s\n", str);
4423 }
4424}
4425
7ef224d1
TZ		 4426static void
4427hist_trigger_entry_print(struct seq_file *m,
4428 struct hist_trigger_data *hist_data, void *key,
4429 struct tracing_map_elt *elt)
4430{
4431 struct hist_field *key_field;
c6afad49 4432 char str[KSYM_SYMBOL_LEN];
69a0200c 4433 bool multiline = false;
85013256 4434 const char *field_name;
7ef224d1
TZ		 4435 unsigned int i;
4436 u64 uval;
4437
4438 seq_puts(m, "{ ");
4439
4440 for_each_hist_key_field(i, hist_data) {
4441 key_field = hist_data->fields[i];
4442
4443 if (i > hist_data->n_vals)
4444 seq_puts(m, ", ");
4445
85013256
TZ		 4446 field_name = hist_field_name(key_field, 0);
4447
0c4a6b46
TZ		 4448 if (key_field->flags & HIST_FIELD_FL_HEX) {
4449 uval = *(u64 *)(key + key_field->offset);
85013256 4450 seq_printf(m, "%s: %llx", field_name, uval);
c6afad49
TZ		 4451 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4452 uval = *(u64 *)(key + key_field->offset);
4453 sprint_symbol_no_offset(str, uval);
85013256
TZ		 4454 seq_printf(m, "%s: [%llx] %-45s", field_name,
4455 uval, str);
c6afad49
TZ		 4456 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4457 uval = *(u64 *)(key + key_field->offset);
4458 sprint_symbol(str, uval);
85013256
TZ		 4459 seq_printf(m, "%s: [%llx] %-55s", field_name,
4460 uval, str);
6b4827ad 4461 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 4462 struct hist_elt_data *elt_data = elt->private_data;
4463 char *comm;
4464
4465 if (WARN_ON_ONCE(!elt_data))
4466 return;
4467
4468 comm = elt_data->comm;
6b4827ad
TZ		 4469
4470 uval = *(u64 *)(key + key_field->offset);
85013256
TZ		 4471 seq_printf(m, "%s: %-16s[%10llu]", field_name,
4472 comm, uval);
31696198
TZ		 4473 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4474 const char *syscall_name;
4475
4476 uval = *(u64 *)(key + key_field->offset);
4477 syscall_name = get_syscall_name(uval);
4478 if (!syscall_name)
4479 syscall_name = "unknown_syscall";
4480
85013256
TZ		 4481 seq_printf(m, "%s: %-30s[%3llu]", field_name,
4482 syscall_name, uval);
69a0200c
TZ		 4483 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4484 seq_puts(m, "stacktrace:\n");
4485 hist_trigger_stacktrace_print(m,
4486 key + key_field->offset,
4487 HIST_STACKTRACE_DEPTH);
4488 multiline = true;
4b94f5b7 4489 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
85013256 4490 seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4b94f5b7 4491 *(u64 *)(key + key_field->offset));
0c4a6b46 4492 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
85013256 4493 seq_printf(m, "%s: %-50s", field_name,
76a3b0c8 4494 (char *)(key + key_field->offset));
7ef224d1 4495 } else {
76a3b0c8 4496 uval = *(u64 *)(key + key_field->offset);
85013256 4497 seq_printf(m, "%s: %10llu", field_name, uval);
7ef224d1
TZ		 4498 }
4499 }
4500
69a0200c
TZ		 4501 if (!multiline)
4502 seq_puts(m, " ");
4503
4504 seq_puts(m, "}");
7ef224d1
TZ		 4505
4506 seq_printf(m, " hitcount: %10llu",
4507 tracing_map_read_sum(elt, HITCOUNT_IDX));
4508
f2606835 4509 for (i = 1; i < hist_data->n_vals; i++) {
85013256
TZ		 4510 field_name = hist_field_name(hist_data->fields[i], 0);
4511
100719dc
TZ		 4512 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4513 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
30350d65
TZ		 4514 continue;
4515
0c4a6b46 4516 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
85013256 4517 seq_printf(m, " %s: %10llx", field_name,
0c4a6b46
TZ		 4518 tracing_map_read_sum(elt, i));
4519 } else {
85013256 4520 seq_printf(m, " %s: %10llu", field_name,
0c4a6b46
TZ		 4521 tracing_map_read_sum(elt, i));
4522 }
f2606835
TZ		 4523 }
4524
50450603
TZ		 4525 print_actions(m, hist_data, elt);
4526
7ef224d1
TZ		 4527 seq_puts(m, "\n");
4528}
4529
4530static int print_entries(struct seq_file *m,
4531 struct hist_trigger_data *hist_data)
4532{
4533 struct tracing_map_sort_entry **sort_entries = NULL;
4534 struct tracing_map *map = hist_data->map;
d50c744e 4535 int i, n_entries;
7ef224d1
TZ		 4536
4537 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4538 hist_data->n_sort_keys,
4539 &sort_entries);
4540 if (n_entries < 0)
4541 return n_entries;
4542
4543 for (i = 0; i < n_entries; i++)
4544 hist_trigger_entry_print(m, hist_data,
4545 sort_entries[i]->key,
4546 sort_entries[i]->elt);
4547
4548 tracing_map_destroy_sort_entries(sort_entries, n_entries);
4549
4550 return n_entries;
4551}
4552
52a7f16d
TZ		 4553static void hist_trigger_show(struct seq_file *m,
4554 struct event_trigger_data *data, int n)
7ef224d1 4555{
7ef224d1 4556 struct hist_trigger_data *hist_data;
6e7a2398 4557 int n_entries;
7ef224d1 4558
52a7f16d
TZ		 4559 if (n > 0)
4560 seq_puts(m, "\n\n");
7ef224d1
TZ		 4561
4562 seq_puts(m, "# event histogram\n#\n# trigger info: ");
4563 data->ops->print(m, data->ops, data);
52a7f16d 4564 seq_puts(m, "#\n\n");
7ef224d1
TZ		 4565
4566 hist_data = data->private_data;
4567 n_entries = print_entries(m, hist_data);
6e7a2398 4568 if (n_entries < 0)
7ef224d1 4569 n_entries = 0;
7ef224d1
TZ		 4570
4571 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
4572 (u64)atomic64_read(&hist_data->map->hits),
4573 n_entries, (u64)atomic64_read(&hist_data->map->drops));
52a7f16d
TZ		 4574}
4575
4576static int hist_show(struct seq_file *m, void *v)
4577{
4578 struct event_trigger_data *data;
4579 struct trace_event_file *event_file;
4580 int n = 0, ret = 0;
4581
4582 mutex_lock(&event_mutex);
4583
4584 event_file = event_file_data(m->private);
4585 if (unlikely(!event_file)) {
4586 ret = -ENODEV;
4587 goto out_unlock;
4588 }
4589
4590 list_for_each_entry_rcu(data, &event_file->triggers, list) {
4591 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4592 hist_trigger_show(m, data, n++);
4593 }
4594
7ef224d1
TZ		 4595 out_unlock:
4596 mutex_unlock(&event_mutex);
4597
4598 return ret;
4599}
4600
4601static int event_hist_open(struct inode *inode, struct file *file)
4602{
4603 return single_open(file, hist_show, file);
4604}
4605
4606const struct file_operations event_hist_fops = {
4607 .open = event_hist_open,
4608 .read = seq_read,
4609 .llseek = seq_lseek,
4610 .release = single_release,
4611};
4612
4613static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
4614{
85013256
TZ		 4615 const char *field_name = hist_field_name(hist_field, 0);
4616
30350d65
TZ		 4617 if (hist_field->var.name)
4618 seq_printf(m, "%s=", hist_field->var.name);
4619
ad42febe
TZ		 4620 if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
4621 seq_puts(m, "common_timestamp");
067fe038
TZ		 4622 else if (field_name) {
4623 if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
4624 seq_putc(m, '$');
ad42febe 4625 seq_printf(m, "%s", field_name);
067fe038 4626 }
ad42febe 4627
0c4a6b46
TZ		 4628 if (hist_field->flags) {
4629 const char *flags_str = get_hist_field_flags(hist_field);
4630
4631 if (flags_str)
4632 seq_printf(m, ".%s", flags_str);
4633 }
7ef224d1
TZ		 4634}
4635
4636static int event_hist_trigger_print(struct seq_file *m,
4637 struct event_trigger_ops *ops,
4638 struct event_trigger_data *data)
4639{
4640 struct hist_trigger_data *hist_data = data->private_data;
30350d65
TZ		 4641 struct hist_field *field;
4642 bool have_var = false;
7ef224d1
TZ		 4643 unsigned int i;
4644
5463bfda
TZ		 4645 seq_puts(m, "hist:");
4646
4647 if (data->name)
4648 seq_printf(m, "%s:", data->name);
4649
4650 seq_puts(m, "keys=");
7ef224d1
TZ		 4651
4652 for_each_hist_key_field(i, hist_data) {
30350d65 4653 field = hist_data->fields[i];
7ef224d1
TZ		 4654
4655 if (i > hist_data->n_vals)
4656 seq_puts(m, ",");
4657
30350d65 4658 if (field->flags & HIST_FIELD_FL_STACKTRACE)
69a0200c
TZ		 4659 seq_puts(m, "stacktrace");
4660 else
30350d65 4661 hist_field_print(m, field);
7ef224d1
TZ		 4662 }
4663
4664 seq_puts(m, ":vals=");
f2606835
TZ		 4665
4666 for_each_hist_val_field(i, hist_data) {
30350d65
TZ		 4667 field = hist_data->fields[i];
4668 if (field->flags & HIST_FIELD_FL_VAR) {
4669 have_var = true;
4670 continue;
4671 }
4672
f2606835
TZ		 4673 if (i == HITCOUNT_IDX)
4674 seq_puts(m, "hitcount");
4675 else {
4676 seq_puts(m, ",");
30350d65
TZ		 4677 hist_field_print(m, field);
4678 }
4679 }
4680
4681 if (have_var) {
4682 unsigned int n = 0;
4683
4684 seq_puts(m, ":");
4685
4686 for_each_hist_val_field(i, hist_data) {
4687 field = hist_data->fields[i];
4688
4689 if (field->flags & HIST_FIELD_FL_VAR) {
4690 if (n++)
4691 seq_puts(m, ",");
4692 hist_field_print(m, field);
4693 }
f2606835
TZ		 4694 }
4695 }
7ef224d1
TZ		 4696
4697 seq_puts(m, ":sort=");
e62347d2
TZ		 4698
4699 for (i = 0; i < hist_data->n_sort_keys; i++) {
4700 struct tracing_map_sort_key *sort_key;
30350d65
TZ		 4701 unsigned int idx, first_key_idx;
4702
4703 /* skip VAR vals */
4704 first_key_idx = hist_data->n_vals - hist_data->n_vars;
e62347d2
TZ		 4705
4706 sort_key = &hist_data->sort_keys[i];
ad42febe
TZ		 4707 idx = sort_key->field_idx;
4708
1a361dfc 4709 if (WARN_ON(idx >= HIST_FIELDS_MAX))
ad42febe 4710 return -EINVAL;
e62347d2
TZ		 4711
4712 if (i > 0)
4713 seq_puts(m, ",");
4714
ad42febe 4715 if (idx == HITCOUNT_IDX)
e62347d2 4716 seq_puts(m, "hitcount");
30350d65
TZ		 4717 else {
4718 if (idx >= first_key_idx)
4719 idx += hist_data->n_vars;
e62347d2 4720 hist_field_print(m, hist_data->fields[idx]);
30350d65 4721 }
e62347d2
TZ		 4722
4723 if (sort_key->descending)
4724 seq_puts(m, ".descending");
4725 }
7ef224d1
TZ		 4726 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
4727
c282a386
TZ		 4728 print_actions_spec(m, hist_data);
4729
7ef224d1
TZ		 4730 if (data->filter_str)
4731 seq_printf(m, " if %s", data->filter_str);
4732
83e99914
TZ		 4733 if (data->paused)
4734 seq_puts(m, " [paused]");
4735 else
4736 seq_puts(m, " [active]");
7ef224d1
TZ		 4737
4738 seq_putc(m, '\n');
4739
4740 return 0;
4741}
4742
5463bfda
TZ		 4743static int event_hist_trigger_init(struct event_trigger_ops *ops,
4744 struct event_trigger_data *data)
4745{
4746 struct hist_trigger_data *hist_data = data->private_data;
4747
4748 if (!data->ref && hist_data->attrs->name)
4749 save_named_trigger(hist_data->attrs->name, data);
4750
4751 data->ref++;
4752
4753 return 0;
4754}
4755
02205a67
TZ		 4756static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
4757{
4758 struct trace_event_file *file;
4759 unsigned int i;
4760 char *cmd;
4761 int ret;
4762
4763 for (i = 0; i < hist_data->n_field_var_hists; i++) {
4764 file = hist_data->field_var_hists[i]->hist_data->event_file;
4765 cmd = hist_data->field_var_hists[i]->cmd;
4766 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
4767 "!hist", "hist", cmd);
4768 }
4769}
4770
7ef224d1
TZ		 4771static void event_hist_trigger_free(struct event_trigger_ops *ops,
4772 struct event_trigger_data *data)
4773{
4774 struct hist_trigger_data *hist_data = data->private_data;
4775
4776 if (WARN_ON_ONCE(data->ref <= 0))
4777 return;
4778
4779 data->ref--;
4780 if (!data->ref) {
5463bfda
TZ		 4781 if (data->name)
4782 del_named_trigger(data);
067fe038 4783
7ef224d1 4784 trigger_data_free(data);
067fe038
TZ		 4785
4786 remove_hist_vars(hist_data);
4787
02205a67
TZ		 4788 unregister_field_var_hists(hist_data);
4789
7ef224d1
TZ		 4790 destroy_hist_data(hist_data);
4791 }
4792}
4793
4794static struct event_trigger_ops event_hist_trigger_ops = {
4795 .func = event_hist_trigger,
4796 .print = event_hist_trigger_print,
5463bfda 4797 .init = event_hist_trigger_init,
7ef224d1
TZ		 4798 .free = event_hist_trigger_free,
4799};
4800
5463bfda
TZ		 4801static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
4802 struct event_trigger_data *data)
4803{
4804 data->ref++;
4805
4806 save_named_trigger(data->named_data->name, data);
4807
4808 event_hist_trigger_init(ops, data->named_data);
4809
4810 return 0;
4811}
4812
4813static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
4814 struct event_trigger_data *data)
4815{
4816 if (WARN_ON_ONCE(data->ref <= 0))
4817 return;
4818
4819 event_hist_trigger_free(ops, data->named_data);
4820
4821 data->ref--;
4822 if (!data->ref) {
4823 del_named_trigger(data);
4824 trigger_data_free(data);
4825 }
4826}
4827
4828static struct event_trigger_ops event_hist_trigger_named_ops = {
4829 .func = event_hist_trigger,
4830 .print = event_hist_trigger_print,
4831 .init = event_hist_trigger_named_init,
4832 .free = event_hist_trigger_named_free,
4833};
4834
7ef224d1
TZ		 4835static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
4836 char *param)
4837{
4838 return &event_hist_trigger_ops;
4839}
4840
e86ae9ba
TZ		 4841static void hist_clear(struct event_trigger_data *data)
4842{
4843 struct hist_trigger_data *hist_data = data->private_data;
e86ae9ba 4844
5463bfda
TZ		 4845 if (data->name)
4846 pause_named_trigger(data);
e86ae9ba
TZ		 4847
4848 synchronize_sched();
4849
4850 tracing_map_clear(hist_data->map);
4851
5463bfda
TZ		 4852 if (data->name)
4853 unpause_named_trigger(data);
4854}
4855
4856static bool compatible_field(struct ftrace_event_field *field,
4857 struct ftrace_event_field *test_field)
4858{
4859 if (field == test_field)
4860 return true;
4861 if (field == NULL || test_field == NULL)
4862 return false;
4863 if (strcmp(field->name, test_field->name) != 0)
4864 return false;
4865 if (strcmp(field->type, test_field->type) != 0)
4866 return false;
4867 if (field->size != test_field->size)
4868 return false;
4869 if (field->is_signed != test_field->is_signed)
4870 return false;
4871
4872 return true;
e86ae9ba
TZ		 4873}
4874
52a7f16d 4875static bool hist_trigger_match(struct event_trigger_data *data,
5463bfda
TZ		 4876 struct event_trigger_data *data_test,
4877 struct event_trigger_data *named_data,
4878 bool ignore_filter)
52a7f16d
TZ		 4879{
4880 struct tracing_map_sort_key *sort_key, *sort_key_test;
4881 struct hist_trigger_data *hist_data, *hist_data_test;
4882 struct hist_field *key_field, *key_field_test;
4883 unsigned int i;
4884
5463bfda
TZ		 4885 if (named_data && (named_data != data_test) &&
4886 (named_data != data_test->named_data))
4887 return false;
4888
4889 if (!named_data && is_named_trigger(data_test))
4890 return false;
4891
52a7f16d
TZ		 4892 hist_data = data->private_data;
4893 hist_data_test = data_test->private_data;
4894
4895 if (hist_data->n_vals != hist_data_test->n_vals ||
4896 hist_data->n_fields != hist_data_test->n_fields ||
4897 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
4898 return false;
4899
5463bfda
TZ		 4900 if (!ignore_filter) {
4901 if ((data->filter_str && !data_test->filter_str) ||
4902 (!data->filter_str && data_test->filter_str))
4903 return false;
4904 }
52a7f16d
TZ		 4905
4906 for_each_hist_field(i, hist_data) {
4907 key_field = hist_data->fields[i];
4908 key_field_test = hist_data_test->fields[i];
4909
4910 if (key_field->flags != key_field_test->flags)
4911 return false;
5463bfda 4912 if (!compatible_field(key_field->field, key_field_test->field))
52a7f16d
TZ		 4913 return false;
4914 if (key_field->offset != key_field_test->offset)
4915 return false;
ad42febe
TZ		 4916 if (key_field->size != key_field_test->size)
4917 return false;
4918 if (key_field->is_signed != key_field_test->is_signed)
4919 return false;
1a361dfc
TZ		 4920 if (!!key_field->var.name != !!key_field_test->var.name)
4921 return false;
4922 if (key_field->var.name &&
4923 strcmp(key_field->var.name, key_field_test->var.name) != 0)
4924 return false;
52a7f16d
TZ		 4925 }
4926
4927 for (i = 0; i < hist_data->n_sort_keys; i++) {
4928 sort_key = &hist_data->sort_keys[i];
4929 sort_key_test = &hist_data_test->sort_keys[i];
4930
4931 if (sort_key->field_idx != sort_key_test->field_idx ||
4932 sort_key->descending != sort_key_test->descending)
4933 return false;
4934 }
4935
5463bfda 4936 if (!ignore_filter && data->filter_str &&
52a7f16d
TZ		 4937 (strcmp(data->filter_str, data_test->filter_str) != 0))
4938 return false;
4939
4940 return true;
4941}
4942
7ef224d1
TZ		 4943static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
4944 struct event_trigger_data *data,
4945 struct trace_event_file *file)
4946{
83e99914 4947 struct hist_trigger_data *hist_data = data->private_data;
5463bfda 4948 struct event_trigger_data *test, *named_data = NULL;
7ef224d1
TZ		 4949 int ret = 0;
4950
5463bfda
TZ		 4951 if (hist_data->attrs->name) {
4952 named_data = find_named_trigger(hist_data->attrs->name);
4953 if (named_data) {
4954 if (!hist_trigger_match(data, named_data, named_data,
4955 true)) {
4956 ret = -EINVAL;
4957 goto out;
4958 }
4959 }
4960 }
4961
4962 if (hist_data->attrs->name && !named_data)
4963 goto new;
4964
7ef224d1
TZ		 4965 list_for_each_entry_rcu(test, &file->triggers, list) {
4966 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 4967 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d 4968 continue;
83e99914
TZ		 4969 if (hist_data->attrs->pause)
4970 test->paused = true;
4971 else if (hist_data->attrs->cont)
4972 test->paused = false;
e86ae9ba
TZ		 4973 else if (hist_data->attrs->clear)
4974 hist_clear(test);
83e99914
TZ		 4975 else
4976 ret = -EEXIST;
7ef224d1
TZ		 4977 goto out;
4978 }
4979 }
5463bfda 4980 new:
e86ae9ba 4981 if (hist_data->attrs->cont || hist_data->attrs->clear) {
83e99914
TZ		 4982 ret = -ENOENT;
4983 goto out;
4984 }
4985
7522c03a
TZ		 4986 if (hist_data->attrs->pause)
4987 data->paused = true;
4988
5463bfda
TZ		 4989 if (named_data) {
4990 destroy_hist_data(data->private_data);
4991 data->private_data = named_data->private_data;
4992 set_named_trigger_data(data, named_data);
4993 data->ops = &event_hist_trigger_named_ops;
4994 }
4995
7ef224d1
TZ		 4996 if (data->ops->init) {
4997 ret = data->ops->init(data->ops, data);
4998 if (ret < 0)
4999 goto out;
5000 }
5001
7ef224d1
TZ		 5002 ret++;
5003
ad42febe
TZ		 5004 if (hist_data->enable_timestamps)
5005 tracing_set_time_stamp_abs(file->tr, true);
067fe038
TZ		 5006 out:
5007 return ret;
5008}
5009
5010static int hist_trigger_enable(struct event_trigger_data *data,
5011 struct trace_event_file *file)
5012{
5013 int ret = 0;
5014
5015 list_add_tail_rcu(&data->list, &file->triggers);
5016
5017 update_cond_flag(file);
ad42febe 5018
7ef224d1
TZ		 5019 if (trace_event_trigger_enable_disable(file, 1) < 0) {
5020 list_del_rcu(&data->list);
5021 update_cond_flag(file);
5022 ret--;
5023 }
067fe038 5024
7ef224d1
TZ		 5025 return ret;
5026}
5027
4b147936
TZ		 5028static bool have_hist_trigger_match(struct event_trigger_data *data,
5029 struct trace_event_file *file)
5030{
5031 struct hist_trigger_data *hist_data = data->private_data;
5032 struct event_trigger_data *test, *named_data = NULL;
5033 bool match = false;
5034
5035 if (hist_data->attrs->name)
5036 named_data = find_named_trigger(hist_data->attrs->name);
5037
5038 list_for_each_entry_rcu(test, &file->triggers, list) {
5039 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5040 if (hist_trigger_match(data, test, named_data, false)) {
5041 match = true;
5042 break;
5043 }
5044 }
5045 }
5046
5047 return match;
5048}
5049
067fe038
TZ		 5050static bool hist_trigger_check_refs(struct event_trigger_data *data,
5051 struct trace_event_file *file)
5052{
5053 struct hist_trigger_data *hist_data = data->private_data;
5054 struct event_trigger_data *test, *named_data = NULL;
5055
5056 if (hist_data->attrs->name)
5057 named_data = find_named_trigger(hist_data->attrs->name);
5058
5059 list_for_each_entry_rcu(test, &file->triggers, list) {
5060 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5061 if (!hist_trigger_match(data, test, named_data, false))
5062 continue;
5063 hist_data = test->private_data;
5064 if (check_var_refs(hist_data))
5065 return true;
5066 break;
5067 }
5068 }
5069
5070 return false;
5071}
5072
52a7f16d
TZ		 5073static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5074 struct event_trigger_data *data,
5075 struct trace_event_file *file)
5076{
5463bfda
TZ		 5077 struct hist_trigger_data *hist_data = data->private_data;
5078 struct event_trigger_data *test, *named_data = NULL;
52a7f16d
TZ		 5079 bool unregistered = false;
5080
5463bfda
TZ		 5081 if (hist_data->attrs->name)
5082 named_data = find_named_trigger(hist_data->attrs->name);
5083
52a7f16d
TZ		 5084 list_for_each_entry_rcu(test, &file->triggers, list) {
5085 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 5086 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d
TZ		 5087 continue;
5088 unregistered = true;
5089 list_del_rcu(&test->list);
5090 trace_event_trigger_enable_disable(file, 0);
5091 update_cond_flag(file);
5092 break;
5093 }
5094 }
5095
5096 if (unregistered && test->ops->free)
5097 test->ops->free(test->ops, test);
ad42febe
TZ		 5098
5099 if (hist_data->enable_timestamps) {
30350d65 5100 if (!hist_data->remove || unregistered)
ad42febe
TZ		 5101 tracing_set_time_stamp_abs(file->tr, false);
5102 }
52a7f16d
TZ		 5103}
5104
067fe038
TZ		 5105static bool hist_file_check_refs(struct trace_event_file *file)
5106{
5107 struct hist_trigger_data *hist_data;
5108 struct event_trigger_data *test;
5109
5110 list_for_each_entry_rcu(test, &file->triggers, list) {
5111 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5112 hist_data = test->private_data;
5113 if (check_var_refs(hist_data))
5114 return true;
5115 }
5116 }
5117
5118 return false;
5119}
5120
52a7f16d
TZ		 5121static void hist_unreg_all(struct trace_event_file *file)
5122{
47c18569 5123 struct event_trigger_data *test, *n;
ad42febe 5124 struct hist_trigger_data *hist_data;
4b147936
TZ		 5125 struct synth_event *se;
5126 const char *se_name;
52a7f16d 5127
067fe038
TZ		 5128 if (hist_file_check_refs(file))
5129 return;
5130
47c18569 5131 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d 5132 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
ad42febe 5133 hist_data = test->private_data;
52a7f16d
TZ		 5134 list_del_rcu(&test->list);
5135 trace_event_trigger_enable_disable(file, 0);
4b147936
TZ		 5136
5137 mutex_lock(&synth_event_mutex);
5138 se_name = trace_event_name(file->event_call);
5139 se = find_synth_event(se_name);
5140 if (se)
5141 se->ref--;
5142 mutex_unlock(&synth_event_mutex);
5143
52a7f16d 5144 update_cond_flag(file);
ad42febe
TZ		 5145 if (hist_data->enable_timestamps)
5146 tracing_set_time_stamp_abs(file->tr, false);
52a7f16d
TZ		 5147 if (test->ops->free)
5148 test->ops->free(test->ops, test);
5149 }
5150 }
5151}
5152
7ef224d1
TZ		 5153static int event_hist_trigger_func(struct event_command *cmd_ops,
5154 struct trace_event_file *file,
5155 char *glob, char *cmd, char *param)
5156{
5157 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5158 struct event_trigger_data *trigger_data;
5159 struct hist_trigger_attrs *attrs;
5160 struct event_trigger_ops *trigger_ops;
5161 struct hist_trigger_data *hist_data;
4b147936
TZ		 5162 struct synth_event *se;
5163 const char *se_name;
30350d65 5164 bool remove = false;
7ef224d1
TZ		 5165 char *trigger;
5166 int ret = 0;
5167
5168 if (!param)
5169 return -EINVAL;
5170
30350d65
TZ		 5171 if (glob[0] == '!')
5172 remove = true;
5173
7ef224d1
TZ		 5174 /* separate the trigger from the filter (k:v [if filter]) */
5175 trigger = strsep(&param, " \t");
5176 if (!trigger)
5177 return -EINVAL;
5178
5179 attrs = parse_hist_trigger_attrs(trigger);
5180 if (IS_ERR(attrs))
5181 return PTR_ERR(attrs);
5182
5183 if (attrs->map_bits)
5184 hist_trigger_bits = attrs->map_bits;
5185
30350d65 5186 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
7ef224d1
TZ		 5187 if (IS_ERR(hist_data)) {
5188 destroy_hist_trigger_attrs(attrs);
5189 return PTR_ERR(hist_data);
5190 }
5191
5192 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5193
7ef224d1 5194 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
4b147936
TZ		 5195 if (!trigger_data) {
5196 ret = -ENOMEM;
7ef224d1 5197 goto out_free;
4b147936 5198 }
7ef224d1
TZ		 5199
5200 trigger_data->count = -1;
5201 trigger_data->ops = trigger_ops;
5202 trigger_data->cmd_ops = cmd_ops;
5203
5204 INIT_LIST_HEAD(&trigger_data->list);
5205 RCU_INIT_POINTER(trigger_data->filter, NULL);
5206
5207 trigger_data->private_data = hist_data;
5208
52a7f16d
TZ		 5209 /* if param is non-empty, it's supposed to be a filter */
5210 if (param && cmd_ops->set_filter) {
5211 ret = cmd_ops->set_filter(param, trigger_data, file);
5212 if (ret < 0)
5213 goto out_free;
5214 }
5215
30350d65 5216 if (remove) {
4b147936
TZ		 5217 if (!have_hist_trigger_match(trigger_data, file))
5218 goto out_free;
5219
067fe038
TZ		 5220 if (hist_trigger_check_refs(trigger_data, file)) {
5221 ret = -EBUSY;
5222 goto out_free;
5223 }
5224
7ef224d1 5225 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
4b147936
TZ		 5226
5227 mutex_lock(&synth_event_mutex);
5228 se_name = trace_event_name(file->event_call);
5229 se = find_synth_event(se_name);
5230 if (se)
5231 se->ref--;
5232 mutex_unlock(&synth_event_mutex);
5233
7ef224d1
TZ		 5234 ret = 0;
5235 goto out_free;
5236 }
5237
7ef224d1
TZ		 5238 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5239 /*
5240 * The above returns on success the # of triggers registered,
5241 * but if it didn't register any it returns zero. Consider no
5242 * triggers registered a failure too.
5243 */
5244 if (!ret) {
e86ae9ba 5245 if (!(attrs->pause || attrs->cont || attrs->clear))
83e99914 5246 ret = -ENOENT;
7ef224d1
TZ		 5247 goto out_free;
5248 } else if (ret < 0)
5249 goto out_free;
067fe038
TZ		 5250
5251 if (get_named_trigger_data(trigger_data))
5252 goto enable;
5253
5254 if (has_hist_vars(hist_data))
5255 save_hist_vars(hist_data);
5256
0212e2aa
TZ		 5257 ret = create_actions(hist_data, file);
5258 if (ret)
5259 goto out_unreg;
5260
067fe038
TZ		 5261 ret = tracing_map_init(hist_data->map);
5262 if (ret)
5263 goto out_unreg;
5264enable:
5265 ret = hist_trigger_enable(trigger_data, file);
5266 if (ret)
5267 goto out_unreg;
5268
4b147936
TZ		 5269 mutex_lock(&synth_event_mutex);
5270 se_name = trace_event_name(file->event_call);
5271 se = find_synth_event(se_name);
5272 if (se)
5273 se->ref++;
5274 mutex_unlock(&synth_event_mutex);
5275
7ef224d1
TZ		 5276 /* Just return zero, not the number of registered triggers */
5277 ret = 0;
5278 out:
5279 return ret;
067fe038
TZ		 5280 out_unreg:
5281 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7ef224d1
TZ		 5282 out_free:
5283 if (cmd_ops->set_filter)
5284 cmd_ops->set_filter(NULL, trigger_data, NULL);
5285
067fe038
TZ		 5286 remove_hist_vars(hist_data);
5287
7ef224d1
TZ		 5288 kfree(trigger_data);
5289
5290 destroy_hist_data(hist_data);
5291 goto out;
5292}
5293
5294static struct event_command trigger_hist_cmd = {
5295 .name = "hist",
5296 .trigger_type = ETT_EVENT_HIST,
5297 .flags = EVENT_CMD_FL_NEEDS_REC,
5298 .func = event_hist_trigger_func,
5299 .reg = hist_register_trigger,
52a7f16d
TZ		 5300 .unreg = hist_unregister_trigger,
5301 .unreg_all = hist_unreg_all,
7ef224d1
TZ		 5302 .get_trigger_ops = event_hist_get_trigger_ops,
5303 .set_filter = set_trigger_filter,
5304};
5305
5306__init int register_trigger_hist_cmd(void)
5307{
5308 int ret;
5309
5310 ret = register_event_command(&trigger_hist_cmd);
5311 WARN_ON(ret < 0);
5312
5313 return ret;
5314}
d0bad49b
TZ		 5315
5316static void
1ac4f51c
TZ		 5317hist_enable_trigger(struct event_trigger_data *data, void *rec,
5318 struct ring_buffer_event *event)
d0bad49b
TZ		 5319{
5320 struct enable_trigger_data *enable_data = data->private_data;
5321 struct event_trigger_data *test;
5322
5323 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
5324 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5325 if (enable_data->enable)
5326 test->paused = false;
5327 else
5328 test->paused = true;
d0bad49b
TZ		 5329 }
5330 }
5331}
5332
5333static void
1ac4f51c
TZ		 5334hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5335 struct ring_buffer_event *event)
d0bad49b
TZ		 5336{
5337 if (!data->count)
5338 return;
5339
5340 if (data->count != -1)
5341 (data->count)--;
5342
1ac4f51c 5343 hist_enable_trigger(data, rec, event);
d0bad49b
TZ		 5344}
5345
5346static struct event_trigger_ops hist_enable_trigger_ops = {
5347 .func = hist_enable_trigger,
5348 .print = event_enable_trigger_print,
5349 .init = event_trigger_init,
5350 .free = event_enable_trigger_free,
5351};
5352
5353static struct event_trigger_ops hist_enable_count_trigger_ops = {
5354 .func = hist_enable_count_trigger,
5355 .print = event_enable_trigger_print,
5356 .init = event_trigger_init,
5357 .free = event_enable_trigger_free,
5358};
5359
5360static struct event_trigger_ops hist_disable_trigger_ops = {
5361 .func = hist_enable_trigger,
5362 .print = event_enable_trigger_print,
5363 .init = event_trigger_init,
5364 .free = event_enable_trigger_free,
5365};
5366
5367static struct event_trigger_ops hist_disable_count_trigger_ops = {
5368 .func = hist_enable_count_trigger,
5369 .print = event_enable_trigger_print,
5370 .init = event_trigger_init,
5371 .free = event_enable_trigger_free,
5372};
5373
5374static struct event_trigger_ops *
5375hist_enable_get_trigger_ops(char *cmd, char *param)
5376{
5377 struct event_trigger_ops *ops;
5378 bool enable;
5379
5380 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5381
5382 if (enable)
5383 ops = param ? &hist_enable_count_trigger_ops :
5384 &hist_enable_trigger_ops;
5385 else
5386 ops = param ? &hist_disable_count_trigger_ops :
5387 &hist_disable_trigger_ops;
5388
5389 return ops;
5390}
5391
52a7f16d
TZ		 5392static void hist_enable_unreg_all(struct trace_event_file *file)
5393{
47c18569 5394 struct event_trigger_data *test, *n;
52a7f16d 5395
47c18569 5396 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d
TZ		 5397 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5398 list_del_rcu(&test->list);
5399 update_cond_flag(file);
5400 trace_event_trigger_enable_disable(file, 0);
5401 if (test->ops->free)
5402 test->ops->free(test->ops, test);
5403 }
5404 }
5405}
5406
d0bad49b
TZ		 5407static struct event_command trigger_hist_enable_cmd = {
5408 .name = ENABLE_HIST_STR,
5409 .trigger_type = ETT_HIST_ENABLE,
5410 .func = event_enable_trigger_func,
5411 .reg = event_enable_register_trigger,
5412 .unreg = event_enable_unregister_trigger,
52a7f16d 5413 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 5414 .get_trigger_ops = hist_enable_get_trigger_ops,
5415 .set_filter = set_trigger_filter,
5416};
5417
5418static struct event_command trigger_hist_disable_cmd = {
5419 .name = DISABLE_HIST_STR,
5420 .trigger_type = ETT_HIST_ENABLE,
5421 .func = event_enable_trigger_func,
5422 .reg = event_enable_register_trigger,
5423 .unreg = event_enable_unregister_trigger,
52a7f16d 5424 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 5425 .get_trigger_ops = hist_enable_get_trigger_ops,
5426 .set_filter = set_trigger_filter,
5427};
5428
5429static __init void unregister_trigger_hist_enable_disable_cmds(void)
5430{
5431 unregister_event_command(&trigger_hist_enable_cmd);
5432 unregister_event_command(&trigger_hist_disable_cmd);
5433}
5434
5435__init int register_trigger_hist_enable_disable_cmds(void)
5436{
5437 int ret;
5438
5439 ret = register_event_command(&trigger_hist_enable_cmd);
5440 if (WARN_ON(ret < 0))
5441 return ret;
5442 ret = register_event_command(&trigger_hist_disable_cmd);
5443 if (WARN_ON(ret < 0))
5444 unregister_trigger_hist_enable_disable_cmds();
5445
5446 return ret;
5447}
4b147936
TZ		 5448
5449static __init int trace_events_hist_init(void)
5450{
5451 struct dentry *entry = NULL;
5452 struct dentry *d_tracer;
5453 int err = 0;
5454
5455 d_tracer = tracing_init_dentry();
5456 if (IS_ERR(d_tracer)) {
5457 err = PTR_ERR(d_tracer);
5458 goto err;
5459 }
5460
5461 entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
5462 NULL, &synth_events_fops);
5463 if (!entry) {
5464 err = -ENODEV;
5465 goto err;
5466 }
5467
5468 return err;
5469 err:
5470 pr_warn("Could not create tracefs 'synthetic_events' entry\n");
5471
5472 return err;
5473}
5474
5475fs_initcall(trace_events_hist_init);
Mirror https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git