watchdog: add sys_info sysctls to dump sys info on system lockup

When soft/hard lockup happens, developers may need different kinds of
system information (call-stacks, memory info, locks, etc.) to help
debugging.

Add 'softlockup_sys_info' and 'hardlockup_sys_info' sysctl knobs to take
human readable string like "tasks,mem,timers,locks,ftrace,...", and when
system lockup happens, all requested information will be printed out.
(refer kernel/sys_info.c for more details).

Link: https://lkml.kernel.org/r/20251113111039.22701-4-feng.tang@linux.alibaba.com
Signed-off-by: Feng Tang <feng.tang@linux.alibaba.com>
Reviewed-by: Petr Mladek <pmladek@suse.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Lance Yang <ioworker0@gmail.com>
Cc: "Paul E . McKenney" <paulmck@kernel.org>
Cc: Petr Mladek <pmladek@suse.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index 45b4408dad3165805cad16ef60cf2d00eff24a4a..176520283f1a258375dc26f91d8b2fcee26b5d2f 100644 (file)
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -582,6 +582,11 @@ if leaking kernel pointer values to unprivileged users is a concern.
 When ``kptr_restrict`` is set to 2, kernel pointers printed using
 %pK will be replaced with 0s regardless of privileges.
 
+softlockup_sys_info & hardlockup_sys_info
+=========================================
+A comma separated list of extra system information to be dumped when
+soft/hard lockup is detected, for example, "tasks,mem,timers,locks,...".
+Refer 'panic_sys_info' section below for more details.
 
 modprobe
 ========

diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 5b62d10027836188dc1c3f4fe7e4ee4bd322b882..1f59b950c475f4f4c4faf15fa5c97f1821b70ac8 100644 (file)
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -25,6 +25,7 @@
 #include <linux/stop_machine.h>
 #include <linux/sysctl.h>
 #include <linux/tick.h>
+#include <linux/sys_info.h>
 
 #include <linux/sched/clock.h>
 #include <linux/sched/debug.h>
@@ -65,6 +66,13 @@ int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
 unsigned int __read_mostly hardlockup_panic =
                        IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);
 
+/*
+ * bitmasks to control what kinds of system info to be printed when
+ * hard lockup is detected, it could be task, memory, lock etc.
+ * Refer include/linux/sys_info.h for detailed bit definition.
+ */
+static unsigned long hardlockup_si_mask;
+
 #ifdef CONFIG_SYSFS
 
 static unsigned int hardlockup_count;
@@ -178,11 +186,15 @@ static void watchdog_hardlockup_kick(void)
 
 void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
 {
+       int hardlockup_all_cpu_backtrace;
+
        if (per_cpu(watchdog_hardlockup_touched, cpu)) {
                per_cpu(watchdog_hardlockup_touched, cpu) = false;
                return;
        }
 
+       hardlockup_all_cpu_backtrace = (hardlockup_si_mask & SYS_INFO_ALL_BT) ?
+                                       1 : sysctl_hardlockup_all_cpu_backtrace;
        /*
         * Check for a hardlockup by making sure the CPU's timer
         * interrupt is incrementing. The timer interrupt should have
@@ -205,7 +217,7 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
                 * Prevent multiple hard-lockup reports if one cpu is already
                 * engaged in dumping all cpu back traces.
                 */
-               if (sysctl_hardlockup_all_cpu_backtrace) {
+               if (hardlockup_all_cpu_backtrace) {
                        if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn))
                                return;
                }
@@ -234,12 +246,13 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
                        trigger_single_cpu_backtrace(cpu);
                }
 
-               if (sysctl_hardlockup_all_cpu_backtrace) {
+               if (hardlockup_all_cpu_backtrace) {
                        trigger_allbutcpu_cpu_backtrace(cpu);
                        if (!hardlockup_panic)
                                clear_bit_unlock(0, &hard_lockup_nmi_warn);
                }
 
+               sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT);
                if (hardlockup_panic)
                        nmi_panic(regs, "Hard LOCKUP");
 
@@ -330,6 +343,13 @@ static void lockup_detector_update_enable(void)
 int __read_mostly sysctl_softlockup_all_cpu_backtrace;
 #endif
 
+/*
+ * bitmasks to control what kinds of system info to be printed when
+ * soft lockup is detected, it could be task, memory, lock etc.
+ * Refer include/linux/sys_info.h for detailed bit definition.
+ */
+static unsigned long softlockup_si_mask;
+
 static struct cpumask watchdog_allowed_mask __read_mostly;
 
 /* Global variables, exported for sysctl */
@@ -746,7 +766,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
        unsigned long touch_ts, period_ts, now;
        struct pt_regs *regs = get_irq_regs();
        int duration;
-       int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
+       int softlockup_all_cpu_backtrace;
        unsigned long flags;
 
        if (!watchdog_enabled)
@@ -758,6 +778,9 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
        if (panic_in_progress())
                return HRTIMER_NORESTART;
 
+       softlockup_all_cpu_backtrace = (softlockup_si_mask & SYS_INFO_ALL_BT) ?
+                                       1 : sysctl_softlockup_all_cpu_backtrace;
+
        watchdog_hardlockup_kick();
 
        /* kick the softlockup detector */
@@ -846,6 +869,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
                }
 
                add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
+               sys_info(softlockup_si_mask & ~SYS_INFO_ALL_BT);
                if (softlockup_panic)
                        panic("softlockup: hung tasks");
        }
@@ -1197,6 +1221,13 @@ static const struct ctl_table watchdog_sysctls[] = {
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
+       {
+               .procname       = "softlockup_sys_info",
+               .data           = &softlockup_si_mask,
+               .maxlen         = sizeof(softlockup_si_mask),
+               .mode           = 0644,
+               .proc_handler   = sysctl_sys_info_handler,
+       },
 #ifdef CONFIG_SMP
        {
                .procname       = "softlockup_all_cpu_backtrace",
@@ -1219,6 +1250,13 @@ static const struct ctl_table watchdog_sysctls[] = {
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
+       {
+               .procname       = "hardlockup_sys_info",
+               .data           = &hardlockup_si_mask,
+               .maxlen         = sizeof(hardlockup_si_mask),
+               .mode           = 0644,
+               .proc_handler   = sysctl_sys_info_handler,
+       },
 #ifdef CONFIG_SMP
        {
                .procname       = "hardlockup_all_cpu_backtrace",