releases/4.14.60/kernel-hung_task.c-show-all-hung-tasks-before-panic.patch

   1 From foo@baz Sat Jul 28 10:25:26 CEST 2018
   2 From: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
   3 Date: Thu, 7 Jun 2018 17:10:34 -0700
   4 Subject: kernel/hung_task.c: show all hung tasks before panic
   5
   6 From: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
   7
   8 [ Upstream commit 401c636a0eeb0d51862fce222da1bf08e3a0ffd0 ]
   9
  10 When we get a hung task it can often be valuable to see _all_ the hung
  11 tasks on the system before calling panic().
  12
  13 Quoting from https://syzkaller.appspot.com/text?tag=CrashReport&id=5316056503549952
  14 ----------------------------------------
  15 INFO: task syz-executor0:6540 blocked for more than 120 seconds.
  16       Not tainted 4.16.0+ #13
  17 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  18 syz-executor0   D23560  6540   4521 0x80000004
  19 Call Trace:
  20  context_switch kernel/sched/core.c:2848 [inline]
  21  __schedule+0x8fb/0x1ef0 kernel/sched/core.c:3490
  22  schedule+0xf5/0x430 kernel/sched/core.c:3549
  23  schedule_preempt_disabled+0x10/0x20 kernel/sched/core.c:3607
  24  __mutex_lock_common kernel/locking/mutex.c:833 [inline]
  25  __mutex_lock+0xb7f/0x1810 kernel/locking/mutex.c:893
  26  mutex_lock_nested+0x16/0x20 kernel/locking/mutex.c:908
  27  lo_ioctl+0x8b/0x1b70 drivers/block/loop.c:1355
  28  __blkdev_driver_ioctl block/ioctl.c:303 [inline]
  29  blkdev_ioctl+0x1759/0x1e00 block/ioctl.c:601
  30  ioctl_by_bdev+0xa5/0x110 fs/block_dev.c:2060
  31  isofs_get_last_session fs/isofs/inode.c:567 [inline]
  32  isofs_fill_super+0x2ba9/0x3bc0 fs/isofs/inode.c:660
  33  mount_bdev+0x2b7/0x370 fs/super.c:1119
  34  isofs_mount+0x34/0x40 fs/isofs/inode.c:1560
  35  mount_fs+0x66/0x2d0 fs/super.c:1222
  36  vfs_kern_mount.part.26+0xc6/0x4a0 fs/namespace.c:1037
  37  vfs_kern_mount fs/namespace.c:2514 [inline]
  38  do_new_mount fs/namespace.c:2517 [inline]
  39  do_mount+0xea4/0x2b90 fs/namespace.c:2847
  40  ksys_mount+0xab/0x120 fs/namespace.c:3063
  41  SYSC_mount fs/namespace.c:3077 [inline]
  42  SyS_mount+0x39/0x50 fs/namespace.c:3074
  43  do_syscall_64+0x281/0x940 arch/x86/entry/common.c:287
  44  entry_SYSCALL_64_after_hwframe+0x42/0xb7
  45 (...snipped...)
  46 Showing all locks held in the system:
  47 (...snipped...)
  48 2 locks held by syz-executor0/6540:
  49  #0: 00000000566d4c39 (&type->s_umount_key#49/1){+.+.}, at: alloc_super fs/super.c:211 [inline]
  50  #0: 00000000566d4c39 (&type->s_umount_key#49/1){+.+.}, at: sget_userns+0x3b2/0xe60 fs/super.c:502 /* down_write_nested(&s->s_umount, SINGLE_DEPTH_NESTING); */
  51  #1: 0000000043ca8836 (&lo->lo_ctl_mutex/1){+.+.}, at: lo_ioctl+0x8b/0x1b70 drivers/block/loop.c:1355 /* mutex_lock_nested(&lo->lo_ctl_mutex, 1); */
  52 (...snipped...)
  53 3 locks held by syz-executor7/6541:
  54  #0: 0000000043ca8836 (&lo->lo_ctl_mutex/1){+.+.}, at: lo_ioctl+0x8b/0x1b70 drivers/block/loop.c:1355 /* mutex_lock_nested(&lo->lo_ctl_mutex, 1); */
  55  #1: 000000007bf3d3f9 (&bdev->bd_mutex){+.+.}, at: blkdev_reread_part+0x1e/0x40 block/ioctl.c:192
  56  #2: 00000000566d4c39 (&type->s_umount_key#50){.+.+}, at: __get_super.part.10+0x1d3/0x280 fs/super.c:663 /* down_read(&sb->s_umount); */
  57 ----------------------------------------
  58
  59 When reporting an AB-BA deadlock like shown above, it would be nice if
  60 trace of PID=6541 is printed as well as trace of PID=6540 before calling
  61 panic().
  62
  63 Showing hung tasks up to /proc/sys/kernel/hung_task_warnings could delay
  64 calling panic() but normally there should not be so many hung tasks.
  65
  66 Link: http://lkml.kernel.org/r/201804050705.BHE57833.HVFOFtSOMQJFOL@I-love.SAKURA.ne.jp
  67 Signed-off-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
  68 Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  69 Acked-by: Dmitry Vyukov <dvyukov@google.com>
  70 Cc: Vegard Nossum <vegard.nossum@oracle.com>
  71 Cc: Mandeep Singh Baines <msb@chromium.org>
  72 Cc: Peter Zijlstra <peterz@infradead.org>
  73 Cc: Thomas Gleixner <tglx@linutronix.de>
  74 Cc: Ingo Molnar <mingo@kernel.org>
  75 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  76
  77 Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
  78
  79 Signed-off-by: Sasha Levin <alexander.levin@microsoft.com>
  80 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  81 ---
  82  kernel/hung_task.c |   11 +++++++----
  83  1 file changed, 7 insertions(+), 4 deletions(-)
  84
  85 --- a/kernel/hung_task.c
  86 +++ b/kernel/hung_task.c
  87 @@ -44,6 +44,7 @@ int __read_mostly sysctl_hung_task_warni
  88
  89  static int __read_mostly did_panic;
  90  static bool hung_task_show_lock;
  91 +static bool hung_task_call_panic;
  92
  93  static struct task_struct *watchdog_task;
  94
  95 @@ -127,10 +128,8 @@ static void check_hung_task(struct task_
  96         touch_nmi_watchdog();
  97
  98         if (sysctl_hung_task_panic) {
  99 -               if (hung_task_show_lock)
 100 -                       debug_show_all_locks();
 101 -               trigger_all_cpu_backtrace();
 102 -               panic("hung_task: blocked tasks");
 103 +               hung_task_show_lock = true;
 104 +               hung_task_call_panic = true;
 105         }
 106  }
 107
 108 @@ -193,6 +192,10 @@ static void check_hung_uninterruptible_t
 109         rcu_read_unlock();
 110         if (hung_task_show_lock)
 111                 debug_show_all_locks();
 112 +       if (hung_task_call_panic) {
 113 +               trigger_all_cpu_backtrace();
 114 +               panic("hung_task: blocked tasks");
 115 +       }
 116  }
 117
 118  static long hung_timeout_jiffies(unsigned long last_checked,