git.ipfire.org Git - thirdparty/kernel/stable-queue.git/blob

   1 From 6eebd5fb20838f5971ba17df9f55cc4f84a31053 Mon Sep 17 00:00:00 2001
   2 From: Waiman Long <longman@redhat.com>
   3 Date: Wed, 22 Jun 2022 16:04:19 -0400
   4 Subject: locking/rwsem: Allow slowpath writer to ignore handoff bit if not set by first waiter
   5
   6 From: Waiman Long <longman@redhat.com>
   7
   8 commit 6eebd5fb20838f5971ba17df9f55cc4f84a31053 upstream.
   9
  10 With commit d257cc8cb8d5 ("locking/rwsem: Make handoff bit handling more
  11 consistent"), the writer that sets the handoff bit can be interrupted
  12 out without clearing the bit if the wait queue isn't empty. This disables
  13 reader and writer optimistic lock spinning and stealing.
  14
  15 Now if a non-first writer in the queue is somehow woken up or a new
  16 waiter enters the slowpath, it can't acquire the lock.  This is not the
  17 case before commit d257cc8cb8d5 as the writer that set the handoff bit
  18 will clear it when exiting out via the out_nolock path. This is less
  19 efficient as the busy rwsem stays in an unlock state for a longer time.
  20
  21 In some cases, this new behavior may cause lockups as shown in [1] and
  22 [2].
  23
  24 This patch allows a non-first writer to ignore the handoff bit if it
  25 is not originally set or initiated by the first waiter. This patch is
  26 shown to be effective in fixing the lockup problem reported in [1].
  27
  28 [1] https://lore.kernel.org/lkml/20220617134325.GC30825@techsingularity.net/
  29 [2] https://lore.kernel.org/lkml/3f02975c-1a9d-be20-32cf-f1d8e3dfafcc@oracle.com/
  30
  31 Fixes: d257cc8cb8d5 ("locking/rwsem: Make handoff bit handling more consistent")
  32 Signed-off-by: Waiman Long <longman@redhat.com>
  33 Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
  34 Acked-by: John Donnelly <john.p.donnelly@oracle.com>
  35 Tested-by: Mel Gorman <mgorman@techsingularity.net>
  36 Link: https://lore.kernel.org/r/20220622200419.778799-1-longman@redhat.com
  37 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  38 ---
  39  kernel/locking/rwsem.c |   30 ++++++++++++++++++++----------
  40  1 file changed, 20 insertions(+), 10 deletions(-)
  41
  42 --- a/kernel/locking/rwsem.c
  43 +++ b/kernel/locking/rwsem.c
  44 @@ -335,8 +335,6 @@ struct rwsem_waiter {
  45         struct task_struct *task;
  46         enum rwsem_waiter_type type;
  47         unsigned long timeout;
  48 -
  49 -       /* Writer only, not initialized in reader */
  50         bool handoff_set;
  51  };
  52  #define rwsem_first_waiter(sem) \
  53 @@ -456,10 +454,12 @@ static void rwsem_mark_wake(struct rw_se
  54                          * to give up the lock), request a HANDOFF to
  55                          * force the issue.
  56                          */
  57 -                       if (!(oldcount & RWSEM_FLAG_HANDOFF) &&
  58 -                           time_after(jiffies, waiter->timeout)) {
  59 -                               adjustment -= RWSEM_FLAG_HANDOFF;
  60 -                               lockevent_inc(rwsem_rlock_handoff);
  61 +                       if (time_after(jiffies, waiter->timeout)) {
  62 +                               if (!(oldcount & RWSEM_FLAG_HANDOFF)) {
  63 +                                       adjustment -= RWSEM_FLAG_HANDOFF;
  64 +                                       lockevent_inc(rwsem_rlock_handoff);
  65 +                               }
  66 +                               waiter->handoff_set = true;
  67                         }
  68
  69                         atomic_long_add(-adjustment, &sem->count);
  70 @@ -569,7 +569,7 @@ static void rwsem_mark_wake(struct rw_se
  71  static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
  72                                         struct rwsem_waiter *waiter)
  73  {
  74 -       bool first = rwsem_first_waiter(sem) == waiter;
  75 +       struct rwsem_waiter *first = rwsem_first_waiter(sem);
  76         long count, new;
  77
  78         lockdep_assert_held(&sem->wait_lock);
  79 @@ -579,11 +579,20 @@ static inline bool rwsem_try_write_lock(
  80                 bool has_handoff = !!(count & RWSEM_FLAG_HANDOFF);
  81
  82                 if (has_handoff) {
  83 -                       if (!first)
  84 +                       /*
  85 +                        * Honor handoff bit and yield only when the first
  86 +                        * waiter is the one that set it. Otherwisee, we
  87 +                        * still try to acquire the rwsem.
  88 +                        */
  89 +                       if (first->handoff_set && (waiter != first))
  90                                 return false;
  91
  92 -                       /* First waiter inherits a previously set handoff bit */
  93 -                       waiter->handoff_set = true;
  94 +                       /*
  95 +                        * First waiter can inherit a previously set handoff
  96 +                        * bit and spin on rwsem if lock acquisition fails.
  97 +                        */
  98 +                       if (waiter == first)
  99 +                               waiter->handoff_set = true;
 100                 }
 101
 102                 new = count;
 103 @@ -978,6 +987,7 @@ queue:
 104         waiter.task = current;
 105         waiter.type = RWSEM_WAITING_FOR_READ;
 106         waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT;
 107 +       waiter.handoff_set = false;
 108
 109         raw_spin_lock_irq(&sem->wait_lock);
 110         if (list_empty(&sem->wait_list)) {