rtla/timerlat: Add continue action

Introduce option to resume tracing after a latency threshold overflow.
The option is implemented as an action named "continue".

Example:
$ rtla timerlat top -q -T 200 -d 1s --on-threshold \
exec,command="echo Threshold" --on-threshold continue
Threshold
Threshold
Threshold
                                     Timer Latency
...

The feature is supported for both hist and top. After the continue
action is executed, processing of the list of actions is stopped and
tracing is resumed.

Cc: John Kacur <jkacur@redhat.com>
Cc: Luis Goncalves <lgoncalv@redhat.com>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Chang Yin <cyin@redhat.com>
Cc: Costa Shulyupin <costa.shul@redhat.com>
Cc: Crystal Wood <crwood@redhat.com>
Cc: Gabriele Monaco <gmonaco@redhat.com>
Link: https://lore.kernel.org/20250626123405.1496931-5-tglozar@redhat.com
Signed-off-by: Tomas Glozar <tglozar@redhat.com>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>

diff --git a/tools/tracing/rtla/src/actions.c b/tools/tracing/rtla/src/actions.c
index 63bee5bdabfdb5576aea43ab4bac29a3d03aac11..aaf0808125d7231f96088fb69a3e9f53d34e5bc4 100644 (file)
--- a/tools/tracing/rtla/src/actions.c
+++ b/tools/tracing/rtla/src/actions.c
@@ -17,6 +17,7 @@ actions_init(struct actions *self)
        self->size = action_default_size;
        self->list = calloc(self->size, sizeof(struct action));
        self->len = 0;
+       self->continue_flag = false;
 
        memset(&self->present, 0, sizeof(self->present));
 
@@ -108,6 +109,20 @@ actions_add_shell(struct actions *self, const char *command)
        return 0;
 }
 
+/*
+ * actions_add_continue - add an action to resume measurement
+ */
+int
+actions_add_continue(struct actions *self)
+{
+       struct action *action = actions_new(self);
+
+       self->present[ACTION_CONTINUE] = true;
+       action->type = ACTION_CONTINUE;
+
+       return 0;
+}
+
 /*
  * actions_parse - add an action based on text specification
  */
@@ -133,6 +148,8 @@ actions_parse(struct actions *self, const char *trigger)
                type = ACTION_SIGNAL;
        else if (strcmp(token, "shell") == 0)
                type = ACTION_SHELL;
+       else if (strcmp(token, "continue") == 0)
+               type = ACTION_CONTINUE;
        else
                /* Invalid trigger type */
                return -1;
@@ -187,6 +204,11 @@ actions_parse(struct actions *self, const char *trigger)
                if (strlen(token) > 8 && strncmp(token, "command=", 8) == 0)
                        return actions_add_shell(self, token + 8);
                return -1;
+       case ACTION_CONTINUE:
+               /* Takes no argument */
+               if (token != NULL)
+                       return -1;
+               return actions_add_continue(self);
        default:
                return -1;
        }
@@ -196,7 +218,7 @@ actions_parse(struct actions *self, const char *trigger)
  * actions_perform - perform all actions
  */
 int
-actions_perform(const struct actions *self)
+actions_perform(struct actions *self)
 {
        int pid, retval;
        const struct action *action;
@@ -226,6 +248,9 @@ actions_perform(const struct actions *self)
                        if (retval)
                                return retval;
                        break;
+               case ACTION_CONTINUE:
+                       self->continue_flag = true;
+                       return 0;
                default:
                        break;
                }

diff --git a/tools/tracing/rtla/src/actions.h b/tools/tracing/rtla/src/actions.h
index 076bbff8519eaa76a02703953d7625d1382f0d80..b10a19d55c49762e06278154c10912dd310ae0a4 100644 (file)
--- a/tools/tracing/rtla/src/actions.h
+++ b/tools/tracing/rtla/src/actions.h
@@ -7,6 +7,7 @@ enum action_type {
        ACTION_TRACE_OUTPUT,
        ACTION_SIGNAL,
        ACTION_SHELL,
+       ACTION_CONTINUE,
        ACTION_FIELD_N
 };
 
@@ -35,6 +36,7 @@ struct actions {
        struct action *list;
        int len, size;
        bool present[ACTION_FIELD_N];
+       bool continue_flag;
 
        /* External dependencies */
        struct tracefs_instance *trace_output_inst;
@@ -45,5 +47,6 @@ void actions_destroy(struct actions *self);
 int actions_add_trace_output(struct actions *self, const char *trace_output);
 int actions_add_signal(struct actions *self, int signal, int pid);
 int actions_add_shell(struct actions *self, const char *command);
+int actions_add_continue(struct actions *self);
 int actions_parse(struct actions *self, const char *trigger);
-int actions_perform(const struct actions *self);
+int actions_perform(struct actions *self);

diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c
index d975d2cd6604d1277cb7d5cc8fffba2d48d23d3f..4f13a8f927116594fbd8bb8528e2683d2a711d68 100644 (file)
--- a/tools/tracing/rtla/src/timerlat_hist.c
+++ b/tools/tracing/rtla/src/timerlat_hist.c
@@ -1374,8 +1374,20 @@ int timerlat_hist_main(int argc, char *argv[])
                                goto out_hist;
                        }
 
-                       if (osnoise_trace_is_off(tool, record))
-                               break;
+                       if (osnoise_trace_is_off(tool, record)) {
+                               actions_perform(&params->actions);
+
+                               if (!params->actions.continue_flag)
+                                       /* continue flag not set, break */
+                                       break;
+
+                               /* continue action reached, re-enable tracing */
+                               if (params->actions.present[ACTION_TRACE_OUTPUT])
+                                       trace_instance_start(&record->trace);
+                               if (!params->no_aa)
+                                       trace_instance_start(&aa->trace);
+                               trace_instance_start(trace);
+                       }
 
                        /* is there still any user-threads ? */
                        if (params->user_workload) {
@@ -1385,8 +1397,27 @@ int timerlat_hist_main(int argc, char *argv[])
                                }
                        }
                }
-       } else
-               timerlat_bpf_wait(-1);
+       } else {
+               while (!stop_tracing) {
+                       timerlat_bpf_wait(-1);
+
+                       if (!stop_tracing) {
+                               /* Threshold overflow, perform actions on threshold */
+                               actions_perform(&params->actions);
+
+                               if (!params->actions.continue_flag)
+                                       /* continue flag not set, break */
+                                       break;
+
+                               /* continue action reached, re-enable tracing */
+                               if (params->actions.present[ACTION_TRACE_OUTPUT])
+                                       trace_instance_start(&record->trace);
+                               if (!params->no_aa)
+                                       trace_instance_start(&aa->trace);
+                               timerlat_bpf_restart_tracing();
+                       }
+               }
+       }
 
        if (params->mode != TRACING_MODE_TRACEFS) {
                timerlat_bpf_detach();
@@ -1412,7 +1443,6 @@ int timerlat_hist_main(int argc, char *argv[])
                if (!params->no_aa)
                        timerlat_auto_analysis(params->stop_us, params->stop_total_us);
 
-               actions_perform(&params->actions);
                return_value = FAILED;
        }
 

diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c
index cdbfda009974912a9f5fd8c7792b7aac18e9d538..60f9c78cb272c64a8604407751c7e0b3db34126f 100644 (file)
--- a/tools/tracing/rtla/src/timerlat_top.c
+++ b/tools/tracing/rtla/src/timerlat_top.c
@@ -906,6 +906,7 @@ timerlat_top_set_signals(struct timerlat_params *params)
 static int
 timerlat_top_main_loop(struct osnoise_tool *top,
                       struct osnoise_tool *record,
+                      struct osnoise_tool *aa,
                       struct timerlat_params *params,
                       struct timerlat_u_params *params_u)
 {
@@ -932,8 +933,20 @@ timerlat_top_main_loop(struct osnoise_tool *top,
                if (!params->quiet)
                        timerlat_print_stats(params, top);
 
-               if (osnoise_trace_is_off(top, record))
-                       break;
+               if (osnoise_trace_is_off(top, record)) {
+                       actions_perform(&params->actions);
+
+                       if (!params->actions.continue_flag)
+                               /* continue flag not set, break */
+                               break;
+
+                       /* continue action reached, re-enable tracing */
+                       if (params->actions.present[ACTION_TRACE_OUTPUT])
+                               trace_instance_start(&record->trace);
+                       if (!params->no_aa)
+                               trace_instance_start(&aa->trace);
+                       trace_instance_start(trace);
+               }
 
                /* is there still any user-threads ? */
                if (params->user_workload) {
@@ -953,6 +966,7 @@ timerlat_top_main_loop(struct osnoise_tool *top,
 static int
 timerlat_top_bpf_main_loop(struct osnoise_tool *top,
                           struct osnoise_tool *record,
+                          struct osnoise_tool *aa,
                           struct timerlat_params *params,
                           struct timerlat_u_params *params_u)
 {
@@ -964,22 +978,9 @@ timerlat_top_bpf_main_loop(struct osnoise_tool *top,
                return 0;
        }
 
-       if (params->quiet) {
-               /* Quiet mode: wait for stop and then, print results */
-               timerlat_bpf_wait(-1);
-
-               retval = timerlat_top_bpf_pull_data(top);
-               if (retval) {
-                       err_msg("Error pulling BPF data\n");
-                       return retval;
-               }
-
-               return 0;
-       }
-
        /* Pull and display data in a loop */
        while (!stop_tracing) {
-               wait_retval = timerlat_bpf_wait(params->sleep_time);
+               wait_retval = timerlat_bpf_wait(params->quiet ? -1 : params->sleep_time);
 
                retval = timerlat_top_bpf_pull_data(top);
                if (retval) {
@@ -987,11 +988,24 @@ timerlat_top_bpf_main_loop(struct osnoise_tool *top,
                        return retval;
                }
 
-               timerlat_print_stats(params, top);
+               if (!params->quiet)
+                       timerlat_print_stats(params, top);
 
-               if (wait_retval == 1)
+               if (wait_retval == 1) {
                        /* Stopping requested by tracer */
-                       break;
+                       actions_perform(&params->actions);
+
+                       if (!params->actions.continue_flag)
+                               /* continue flag not set, break */
+                               break;
+
+                       /* continue action reached, re-enable tracing */
+                       if (params->actions.present[ACTION_TRACE_OUTPUT])
+                               trace_instance_start(&record->trace);
+                       if (!params->no_aa)
+                               trace_instance_start(&aa->trace);
+                       timerlat_bpf_restart_tracing();
+               }
 
                /* is there still any user-threads ? */
                if (params->user_workload) {
@@ -1205,9 +1219,9 @@ int timerlat_top_main(int argc, char *argv[])
        timerlat_top_set_signals(params);
 
        if (params->mode == TRACING_MODE_TRACEFS)
-               retval = timerlat_top_main_loop(top, record, params, &params_u);
+               retval = timerlat_top_main_loop(top, record, aa, params, &params_u);
        else
-               retval = timerlat_top_bpf_main_loop(top, record, params, &params_u);
+               retval = timerlat_top_bpf_main_loop(top, record, aa, params, &params_u);
 
        if (retval)
                goto out_top;
@@ -1230,7 +1244,6 @@ int timerlat_top_main(int argc, char *argv[])
                if (!params->no_aa)
                        timerlat_auto_analysis(params->stop_us, params->stop_total_us);
 
-               actions_perform(&params->actions);
                return_value = FAILED;
        } else if (params->aa_only) {
                /*