queue-4.19/sched-core-use-read_once-write_once-in-move_queued_t.patch

   1 From fc823c03daae792612412a48bd8eb5ee6f4cbbf7 Mon Sep 17 00:00:00 2001
   2 From: Andrea Parri <andrea.parri@amarulasolutions.com>
   3 Date: Mon, 21 Jan 2019 16:52:40 +0100
   4 Subject: sched/core: Use READ_ONCE()/WRITE_ONCE() in
   5  move_queued_task()/task_rq_lock()
   6
   7 [ Upstream commit c546951d9c9300065bad253ecdf1ac59ce9d06c8 ]
   8
   9 move_queued_task() synchronizes with task_rq_lock() as follows:
  10
  11         move_queued_task()              task_rq_lock()
  12
  13         [S] ->on_rq = MIGRATING         [L] rq = task_rq()
  14         WMB (__set_task_cpu())          ACQUIRE (rq->lock);
  15         [S] ->cpu = new_cpu             [L] ->on_rq
  16
  17 where "[L] rq = task_rq()" is ordered before "ACQUIRE (rq->lock)" by an
  18 address dependency and, in turn, "ACQUIRE (rq->lock)" is ordered before
  19 "[L] ->on_rq" by the ACQUIRE itself.
  20
  21 Use READ_ONCE() to load ->cpu in task_rq() (c.f., task_cpu()) to honor
  22 this address dependency.  Also, mark the accesses to ->cpu and ->on_rq
  23 with READ_ONCE()/WRITE_ONCE() to comply with the LKMM.
  24
  25 Signed-off-by: Andrea Parri <andrea.parri@amarulasolutions.com>
  26 Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
  27 Cc: Alan Stern <stern@rowland.harvard.edu>
  28 Cc: Linus Torvalds <torvalds@linux-foundation.org>
  29 Cc: Mike Galbraith <efault@gmx.de>
  30 Cc: Paul E. McKenney <paulmck@linux.ibm.com>
  31 Cc: Peter Zijlstra <peterz@infradead.org>
  32 Cc: Thomas Gleixner <tglx@linutronix.de>
  33 Cc: Will Deacon <will.deacon@arm.com>
  34 Link: https://lkml.kernel.org/r/20190121155240.27173-1-andrea.parri@amarulasolutions.com
  35 Signed-off-by: Ingo Molnar <mingo@kernel.org>
  36 Signed-off-by: Sasha Levin <sashal@kernel.org>
  37 ---
  38  include/linux/sched.h | 4 ++--
  39  kernel/sched/core.c   | 9 +++++----
  40  kernel/sched/sched.h  | 6 +++---
  41  3 files changed, 10 insertions(+), 9 deletions(-)
  42
  43 diff --git a/include/linux/sched.h b/include/linux/sched.h
  44 index 4abb5bd74b04..5dc024e28397 100644
  45 --- a/include/linux/sched.h
  46 +++ b/include/linux/sched.h
  47 @@ -1737,9 +1737,9 @@ static __always_inline bool need_resched(void)
  48  static inline unsigned int task_cpu(const struct task_struct *p)
  49  {
  50  #ifdef CONFIG_THREAD_INFO_IN_TASK
  51 -       return p->cpu;
  52 +       return READ_ONCE(p->cpu);
  53  #else
  54 -       return task_thread_info(p)->cpu;
  55 +       return READ_ONCE(task_thread_info(p)->cpu);
  56  #endif
  57  }
  58
  59 diff --git a/kernel/sched/core.c b/kernel/sched/core.c
  60 index 152a0b0c91bb..9a4f57d7e931 100644
  61 --- a/kernel/sched/core.c
  62 +++ b/kernel/sched/core.c
  63 @@ -107,11 +107,12 @@ struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
  64                  *                                      [L] ->on_rq
  65                  *      RELEASE (rq->lock)
  66                  *
  67 -                * If we observe the old CPU in task_rq_lock, the acquire of
  68 +                * If we observe the old CPU in task_rq_lock(), the acquire of
  69                  * the old rq->lock will fully serialize against the stores.
  70                  *
  71 -                * If we observe the new CPU in task_rq_lock, the acquire will
  72 -                * pair with the WMB to ensure we must then also see migrating.
  73 +                * If we observe the new CPU in task_rq_lock(), the address
  74 +                * dependency headed by '[L] rq = task_rq()' and the acquire
  75 +                * will pair with the WMB to ensure we then also see migrating.
  76                  */
  77                 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
  78                         rq_pin_lock(rq, rf);
  79 @@ -910,7 +911,7 @@ static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
  80  {
  81         lockdep_assert_held(&rq->lock);
  82
  83 -       p->on_rq = TASK_ON_RQ_MIGRATING;
  84 +       WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
  85         dequeue_task(rq, p, DEQUEUE_NOCLOCK);
  86         set_task_cpu(p, new_cpu);
  87         rq_unlock(rq, rf);
  88 diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
  89 index b63172288f7b..4c7a837d7c14 100644
  90 --- a/kernel/sched/sched.h
  91 +++ b/kernel/sched/sched.h
  92 @@ -1331,9 +1331,9 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
  93          */
  94         smp_wmb();
  95  #ifdef CONFIG_THREAD_INFO_IN_TASK
  96 -       p->cpu = cpu;
  97 +       WRITE_ONCE(p->cpu, cpu);
  98  #else
  99 -       task_thread_info(p)->cpu = cpu;
 100 +       WRITE_ONCE(task_thread_info(p)->cpu, cpu);
 101  #endif
 102         p->wake_cpu = cpu;
 103  #endif
 104 @@ -1434,7 +1434,7 @@ static inline int task_on_rq_queued(struct task_struct *p)
 105
 106  static inline int task_on_rq_migrating(struct task_struct *p)
 107  {
 108 -       return p->on_rq == TASK_ON_RQ_MIGRATING;
 109 +       return READ_ONCE(p->on_rq) == TASK_ON_RQ_MIGRATING;
 110  }
 111
 112  /*
 113 --
 114 2.19.1
 115