MINOR: debug: add "debug dev trace" to flood with traces

This new command, enabled only with "DEBUG_DEV", sends 2 or 20 traces
per task wakeup (depending on the verbosity level), and stops after 1M
wakeups per thread in order not to have to stop/start the process each
time it's fired.

We have two small messages and 18 larger ones from 20 to 270 bytes
each, so that the average size is approx 213 bytes counting headers
(the header adds approx 82 bytes), which matches what's generally
observed on average when traces are enabled in all muxes.

Typical figures show varations between 5.7M and 6.2M msg/s on an EPYC
in a 3C6T setup (single CCX), and 2.12M - 2.22M in a 24C48T setup
(across 8 CCX, with 8 thread groups).

diff --git a/src/debug.c b/src/debug.c
index 3ab5feb84fa80d12108c68c2bf9c8d1815ed4058..077b0377f46df3bc25d047037e4b75fdb8483b47 100644 (file)
--- a/src/debug.c
+++ b/src/debug.c
@@ -46,6 +46,7 @@
 #include <haproxy/thread.h>
 #include <haproxy/time.h>
 #include <haproxy/tools.h>
+#include <haproxy/trace.h>
 #include <import/ist.h>
 
 
@@ -1504,6 +1505,112 @@ static int debug_parse_cli_sched(char **args, char *payload, struct appctx *appc
        return cli_err(appctx, "Not enough memory");
 }
 
+#if defined(DEBUG_DEV)
+/* All of this is for "trace dbg" */
+
+static struct trace_source trace_dbg __read_mostly = {
+       .name = IST("dbg"),
+       .desc = "trace debugger",
+       .report_events = ~0,  // report everything by default
+};
+
+#define TRACE_SOURCE &trace_dbg
+INITCALL1(STG_REGISTER, trace_register_source, TRACE_SOURCE);
+
+/* This is the task handler used to send traces in loops. Note that the task's
+ * context contains the number of remaining calls to be done. The task sends 20
+ * messages per wakeup.
+ */
+static struct task *debug_trace_task(struct task *t, void *ctx, unsigned int state)
+{
+       ulong count;
+
+       /* send 2 traces enter/leave +18 devel = 20 traces total */
+       TRACE_ENTER(1);
+       TRACE_DEVEL("msg01 has 20 bytes .", 1);
+       TRACE_DEVEL("msg02 has 20 bytes .", 1);
+       TRACE_DEVEL("msg03 has 20 bytes .", 1);
+       TRACE_DEVEL("msg04 has 70 bytes payload: 0123456789 0123456789 0123456789 012345678", 1);
+       TRACE_DEVEL("msg05 has 70 bytes payload: 0123456789 0123456789 0123456789 012345678", 1);
+       TRACE_DEVEL("msg06 has 70 bytes payload: 0123456789 0123456789 0123456789 012345678", 1);
+       TRACE_DEVEL("msg07 has 120 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 012", 1);
+       TRACE_DEVEL("msg08 has 120 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 012", 1);
+       TRACE_DEVEL("msg09 has 120 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 012", 1);
+       TRACE_DEVEL("msg10 has 170 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 012345678", 1);
+       TRACE_DEVEL("msg11 has 170 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 012345678", 1);
+       TRACE_DEVEL("msg12 has 170 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 012345678", 1);
+       TRACE_DEVEL("msg13 has 220 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123", 1);
+       TRACE_DEVEL("msg14 has 220 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123", 1);
+       TRACE_DEVEL("msg15 has 220 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123", 1);
+       TRACE_DEVEL("msg16 has 270 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789", 1);
+       TRACE_DEVEL("msg17 has 270 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789", 1);
+       TRACE_DEVEL("msg18 has 270 bytes payload: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789", 1);
+       TRACE_LEAVE(1);
+
+       count = (ulong)t->context;
+       t->context = (void*)count - 1;
+
+       if (count)
+               task_wakeup(t, TASK_WOKEN_MSG);
+       else {
+               task_destroy(t);
+               t = NULL;
+       }
+       return t;
+}
+
+/* parse a "debug dev trace" command
+ * debug dev trace <nbthr>.
+ * It will create as many tasks (one per thread), starting from lowest threads.
+ * The traces will stop after 1M wakeups or 20M messages ~= 4GB of data.
+ */
+static int debug_parse_cli_trace(char **args, char *payload, struct appctx *appctx, void *private)
+{
+       unsigned long count = 1;
+       unsigned long i;
+       char *msg = NULL;
+       char *endarg;
+
+       if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
+               return 1;
+
+       _HA_ATOMIC_INC(&debug_commands_issued);
+
+       if (!args[3][0]) {
+               memprintf(&msg, "Need a thread count. Note that 20M msg will be sent per thread.\n");
+               goto fail;
+       }
+
+       /* parse the new value . */
+       count = strtoll(args[3], &endarg, 0);
+       if (args[3][1] && *endarg) {
+               memprintf(&msg, "Ignoring unparsable thread number '%s'.\n", args[3]);
+               goto fail;
+       }
+
+       if (count >= global.nbthread)
+               count = global.nbthread;
+
+       for (i = 0; i < count; i++) {
+               struct task *task = task_new_on(i);
+
+               if (!task)
+                       goto fail;
+
+               task->process = debug_trace_task;
+               task->context = (void*)(ulong)1000000; // 1M wakeups = 20M messages
+               task_wakeup(task, TASK_WOKEN_INIT);
+       }
+
+       if (msg && *msg)
+               return cli_dynmsg(appctx, LOG_INFO, msg);
+       return 1;
+
+ fail:
+       return cli_dynmsg(appctx, LOG_ERR, msg);
+}
+#endif /* DEBUG_DEV */
+
 /* CLI state for "debug dev fd" */
 struct dev_fd_ctx {
        int start_fd;
@@ -2295,6 +2402,9 @@ static struct cli_kw_list cli_kws = {{ },{
        {{ "debug", "dev", "sym",   NULL },    "debug dev sym    <addr>                 : resolve symbol address",                  debug_parse_cli_sym,   NULL, NULL, NULL, ACCESS_EXPERT },
        {{ "debug", "dev", "task",  NULL },    "debug dev task <ptr> [wake|expire|kill] : show/wake/expire/kill task/tasklet",      debug_parse_cli_task,  NULL, NULL, NULL, ACCESS_EXPERT },
        {{ "debug", "dev", "tkill", NULL },    "debug dev tkill  [thr] [sig]            : send signal to thread",                   debug_parse_cli_tkill, NULL, NULL, NULL, ACCESS_EXPERT },
+#if defined(DEBUG_DEV)
+       {{ "debug", "dev", "trace", NULL },    "debug dev trace [nbthr]                 : flood traces from that many threads",     debug_parse_cli_trace,  NULL, NULL, NULL, ACCESS_EXPERT },
+#endif
        {{ "debug", "dev", "warn",  NULL },    "debug dev warn                          : call WARN_ON() and possibly crash",       debug_parse_cli_warn,  NULL, NULL, NULL, ACCESS_EXPERT },
        {{ "debug", "dev", "write", NULL },    "debug dev write  [size]                 : write that many bytes in return",         debug_parse_cli_write, NULL, NULL, NULL, ACCESS_EXPERT },