--- /dev/null
+From foo@baz Thu Mar 4 02:09:29 PM CET 2021
+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Mon, 1 Mar 2021 18:31:22 +0100
+Subject: futex: Cleanup refcounting
+To: stable@vger.kernel.org
+Cc: Lee Jones <lee.jones@linaro.org>, "Luis Claudio R. Goncalves" <lgoncalv@redhat.com>
+Message-ID: <YD0k6ppZS2lYYEqg@decadent.org.uk>
+Content-Disposition: inline
+
+From: Ben Hutchings <ben@decadent.org.uk>
+
+From: Peter Zijlstra <peterz@infradead.org>
+
+commit bf92cf3a5100f5a0d5f9834787b130159397cb22 upstream.
+
+Add a put_pit_state() as counterpart for get_pi_state() so the refcounting
+becomes consistent.
+
+Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Cc: juri.lelli@arm.com
+Cc: bigeasy@linutronix.de
+Cc: xlpang@redhat.com
+Cc: rostedt@goodmis.org
+Cc: mathieu.desnoyers@efficios.com
+Cc: jdesfossez@efficios.com
+Cc: dvhart@infradead.org
+Cc: bristot@redhat.com
+Link: http://lkml.kernel.org/r/20170322104151.801778516@infradead.org
+Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
+Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ kernel/futex.c | 13 +++++++++----
+ 1 file changed, 9 insertions(+), 4 deletions(-)
+
+--- a/kernel/futex.c
++++ b/kernel/futex.c
+@@ -827,7 +827,7 @@ static int refill_pi_state_cache(void)
+ return 0;
+ }
+
+-static struct futex_pi_state * alloc_pi_state(void)
++static struct futex_pi_state *alloc_pi_state(void)
+ {
+ struct futex_pi_state *pi_state = current->pi_state_cache;
+
+@@ -860,6 +860,11 @@ static void pi_state_update_owner(struct
+ }
+ }
+
++static void get_pi_state(struct futex_pi_state *pi_state)
++{
++ WARN_ON_ONCE(!atomic_inc_not_zero(&pi_state->refcount));
++}
++
+ /*
+ * Drops a reference to the pi_state object and frees or caches it
+ * when the last reference is gone.
+@@ -901,7 +906,7 @@ static void put_pi_state(struct futex_pi
+ * Look up the task based on what TID userspace gave us.
+ * We dont trust it.
+ */
+-static struct task_struct * futex_find_get_task(pid_t pid)
++static struct task_struct *futex_find_get_task(pid_t pid)
+ {
+ struct task_struct *p;
+
+@@ -1149,7 +1154,7 @@ static int attach_to_pi_state(u32 __user
+ goto out_einval;
+
+ out_attach:
+- atomic_inc(&pi_state->refcount);
++ get_pi_state(pi_state);
+ raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
+ *ps = pi_state;
+ return 0;
+@@ -2210,7 +2215,7 @@ retry_private:
+ * refcount on the pi_state and store the pointer in
+ * the futex_q object of the waiter.
+ */
+- atomic_inc(&pi_state->refcount);
++ get_pi_state(pi_state);
+ this->pi_state = pi_state;
+ ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
+ this->rt_waiter,
--- /dev/null
+From foo@baz Thu Mar 4 02:09:29 PM CET 2021
+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Mon, 1 Mar 2021 18:30:39 +0100
+Subject: futex: Cleanup variable names for futex_top_waiter()
+To: stable@vger.kernel.org
+Cc: Lee Jones <lee.jones@linaro.org>, "Luis Claudio R. Goncalves" <lgoncalv@redhat.com>
+Message-ID: <YD0kv9H996Tkhg2o@decadent.org.uk>
+Content-Disposition: inline
+
+From: Ben Hutchings <ben@decadent.org.uk>
+
+From: Peter Zijlstra <peterz@infradead.org>
+
+commit 499f5aca2cdd5e958b27e2655e7e7f82524f46b1 upstream.
+
+futex_top_waiter() returns the top-waiter on the pi_mutex. Assinging
+this to a variable 'match' totally obscures the code.
+
+Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Cc: juri.lelli@arm.com
+Cc: bigeasy@linutronix.de
+Cc: xlpang@redhat.com
+Cc: rostedt@goodmis.org
+Cc: mathieu.desnoyers@efficios.com
+Cc: jdesfossez@efficios.com
+Cc: dvhart@infradead.org
+Cc: bristot@redhat.com
+Link: http://lkml.kernel.org/r/20170322104151.554710645@infradead.org
+Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
+[bwh: Backported to 4.9: adjust context]
+Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ kernel/futex.c | 28 ++++++++++++++--------------
+ 1 file changed, 14 insertions(+), 14 deletions(-)
+
+--- a/kernel/futex.c
++++ b/kernel/futex.c
+@@ -1352,14 +1352,14 @@ static int lookup_pi_state(u32 __user *u
+ union futex_key *key, struct futex_pi_state **ps,
+ struct task_struct **exiting)
+ {
+- struct futex_q *match = futex_top_waiter(hb, key);
++ struct futex_q *top_waiter = futex_top_waiter(hb, key);
+
+ /*
+ * If there is a waiter on that futex, validate it and
+ * attach to the pi_state when the validation succeeds.
+ */
+- if (match)
+- return attach_to_pi_state(uaddr, uval, match->pi_state, ps);
++ if (top_waiter)
++ return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
+
+ /*
+ * We are the first waiter - try to look up the owner based on
+@@ -1414,7 +1414,7 @@ static int futex_lock_pi_atomic(u32 __us
+ int set_waiters)
+ {
+ u32 uval, newval, vpid = task_pid_vnr(task);
+- struct futex_q *match;
++ struct futex_q *top_waiter;
+ int ret;
+
+ /*
+@@ -1440,9 +1440,9 @@ static int futex_lock_pi_atomic(u32 __us
+ * Lookup existing state first. If it exists, try to attach to
+ * its pi_state.
+ */
+- match = futex_top_waiter(hb, key);
+- if (match)
+- return attach_to_pi_state(uaddr, uval, match->pi_state, ps);
++ top_waiter = futex_top_waiter(hb, key);
++ if (top_waiter)
++ return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
+
+ /*
+ * No waiter and user TID is 0. We are here because the
+@@ -1532,11 +1532,11 @@ static void mark_wake_futex(struct wake_
+ q->lock_ptr = NULL;
+ }
+
+-static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this,
++static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *top_waiter,
+ struct futex_hash_bucket *hb)
+ {
+ struct task_struct *new_owner;
+- struct futex_pi_state *pi_state = this->pi_state;
++ struct futex_pi_state *pi_state = top_waiter->pi_state;
+ u32 uninitialized_var(curval), newval;
+ WAKE_Q(wake_q);
+ bool deboost;
+@@ -1557,7 +1557,7 @@ static int wake_futex_pi(u32 __user *uad
+
+ /*
+ * When we interleave with futex_lock_pi() where it does
+- * rt_mutex_timed_futex_lock(), we might observe @this futex_q waiter,
++ * rt_mutex_timed_futex_lock(), we might observe @top_waiter futex_q waiter,
+ * but the rt_mutex's wait_list can be empty (either still, or again,
+ * depending on which side we land).
+ *
+@@ -2975,7 +2975,7 @@ static int futex_unlock_pi(u32 __user *u
+ u32 uninitialized_var(curval), uval, vpid = task_pid_vnr(current);
+ union futex_key key = FUTEX_KEY_INIT;
+ struct futex_hash_bucket *hb;
+- struct futex_q *match;
++ struct futex_q *top_waiter;
+ int ret;
+
+ retry:
+@@ -2999,9 +2999,9 @@ retry:
+ * all and we at least want to know if user space fiddled
+ * with the futex value instead of blindly unlocking.
+ */
+- match = futex_top_waiter(hb, &key);
+- if (match) {
+- ret = wake_futex_pi(uaddr, uval, match, hb);
++ top_waiter = futex_top_waiter(hb, &key);
++ if (top_waiter) {
++ ret = wake_futex_pi(uaddr, uval, top_waiter, hb);
+ /*
+ * In case of success wake_futex_pi dropped the hash
+ * bucket lock.
--- /dev/null
+From foo@baz Thu Mar 4 02:09:29 PM CET 2021
+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Mon, 1 Mar 2021 18:32:02 +0100
+Subject: futex: Don't enable IRQs unconditionally in put_pi_state()
+To: stable@vger.kernel.org
+Cc: Lee Jones <lee.jones@linaro.org>, "Luis Claudio R. Goncalves" <lgoncalv@redhat.com>
+Message-ID: <YD0lErDoywqSJyM8@decadent.org.uk>
+Content-Disposition: inline
+
+From: Ben Hutchings <ben@decadent.org.uk>
+
+From: Dan Carpenter <dan.carpenter@oracle.com>
+
+commit 1e106aa3509b86738769775969822ffc1ec21bf4 upstream.
+
+The exit_pi_state_list() function calls put_pi_state() with IRQs disabled
+and is not expecting that IRQs will be enabled inside the function.
+
+Use the _irqsave() variant so that IRQs are restored to the original state
+instead of being enabled unconditionally.
+
+Fixes: 153fbd1226fb ("futex: Fix more put_pi_state() vs. exit_pi_state_list() races")
+Signed-off-by: Dan Carpenter <dan.carpenter@oracle.com>
+Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
+Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Cc: stable@vger.kernel.org
+Link: https://lore.kernel.org/r/20201106085205.GA1159983@mwanda
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+[bwh: Backported to 4.9: adjust context]
+Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ kernel/futex.c | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+
+--- a/kernel/futex.c
++++ b/kernel/futex.c
+@@ -882,10 +882,12 @@ static void put_pi_state(struct futex_pi
+ * and has cleaned up the pi_state already
+ */
+ if (pi_state->owner) {
+- raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
++ unsigned long flags;
++
++ raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags);
+ pi_state_update_owner(pi_state, NULL);
+ rt_mutex_proxy_unlock(&pi_state->pi_mutex);
+- raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
++ raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);
+ }
+
+ if (current->pi_state_cache) {
--- /dev/null
+From foo@baz Thu Mar 4 02:09:29 PM CET 2021
+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Mon, 1 Mar 2021 18:31:55 +0100
+Subject: futex: Fix more put_pi_state() vs. exit_pi_state_list() races
+To: stable@vger.kernel.org
+Cc: Lee Jones <lee.jones@linaro.org>, "Luis Claudio R. Goncalves" <lgoncalv@redhat.com>
+Message-ID: <YD0lC+fuTMO3BKSw@decadent.org.uk>
+Content-Disposition: inline
+
+From: Ben Hutchings <ben@decadent.org.uk>
+
+From: Peter Zijlstra <peterz@infradead.org>
+
+commit 153fbd1226fb30b8630802aa5047b8af5ef53c9f upstream.
+
+Dmitry (through syzbot) reported being able to trigger the WARN in
+get_pi_state() and a use-after-free on:
+
+ raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
+
+Both are due to this race:
+
+ exit_pi_state_list() put_pi_state()
+
+ lock(&curr->pi_lock)
+ while() {
+ pi_state = list_first_entry(head);
+ hb = hash_futex(&pi_state->key);
+ unlock(&curr->pi_lock);
+
+ dec_and_test(&pi_state->refcount);
+
+ lock(&hb->lock)
+ lock(&pi_state->pi_mutex.wait_lock) // uaf if pi_state free'd
+ lock(&curr->pi_lock);
+
+ ....
+
+ unlock(&curr->pi_lock);
+ get_pi_state(); // WARN; refcount==0
+
+The problem is we take the reference count too late, and don't allow it
+being 0. Fix it by using inc_not_zero() and simply retrying the loop
+when we fail to get a refcount. In that case put_pi_state() should
+remove the entry from the list.
+
+Reported-by: Dmitry Vyukov <dvyukov@google.com>
+Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
+Cc: Gratian Crisan <gratian.crisan@ni.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: dvhart@infradead.org
+Cc: syzbot <bot+2af19c9e1ffe4d4ee1d16c56ae7580feaee75765@syzkaller.appspotmail.com>
+Cc: syzkaller-bugs@googlegroups.com
+Cc: <stable@vger.kernel.org>
+Fixes: c74aef2d06a9 ("futex: Fix pi_state->owner serialization")
+Link: http://lkml.kernel.org/r/20171031101853.xpfh72y643kdfhjs@hirez.programming.kicks-ass.net
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ kernel/futex.c | 23 ++++++++++++++++++++---
+ 1 file changed, 20 insertions(+), 3 deletions(-)
+
+--- a/kernel/futex.c
++++ b/kernel/futex.c
+@@ -941,11 +941,27 @@ static void exit_pi_state_list(struct ta
+ */
+ raw_spin_lock_irq(&curr->pi_lock);
+ while (!list_empty(head)) {
+-
+ next = head->next;
+ pi_state = list_entry(next, struct futex_pi_state, list);
+ key = pi_state->key;
+ hb = hash_futex(&key);
++
++ /*
++ * We can race against put_pi_state() removing itself from the
++ * list (a waiter going away). put_pi_state() will first
++ * decrement the reference count and then modify the list, so
++ * its possible to see the list entry but fail this reference
++ * acquire.
++ *
++ * In that case; drop the locks to let put_pi_state() make
++ * progress and retry the loop.
++ */
++ if (!atomic_inc_not_zero(&pi_state->refcount)) {
++ raw_spin_unlock_irq(&curr->pi_lock);
++ cpu_relax();
++ raw_spin_lock_irq(&curr->pi_lock);
++ continue;
++ }
+ raw_spin_unlock_irq(&curr->pi_lock);
+
+ spin_lock(&hb->lock);
+@@ -956,8 +972,10 @@ static void exit_pi_state_list(struct ta
+ * task still owns the PI-state:
+ */
+ if (head->next != next) {
++ /* retain curr->pi_lock for the loop invariant */
+ raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
+ spin_unlock(&hb->lock);
++ put_pi_state(pi_state);
+ continue;
+ }
+
+@@ -965,9 +983,8 @@ static void exit_pi_state_list(struct ta
+ WARN_ON(list_empty(&pi_state->list));
+ list_del_init(&pi_state->list);
+ pi_state->owner = NULL;
+- raw_spin_unlock(&curr->pi_lock);
+
+- get_pi_state(pi_state);
++ raw_spin_unlock(&curr->pi_lock);
+ raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
+ spin_unlock(&hb->lock);
+
--- /dev/null
+From foo@baz Thu Mar 4 02:09:29 PM CET 2021
+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Mon, 1 Mar 2021 18:31:48 +0100
+Subject: futex: Fix pi_state->owner serialization
+To: stable@vger.kernel.org
+Cc: Lee Jones <lee.jones@linaro.org>, "Luis Claudio R. Goncalves" <lgoncalv@redhat.com>
+Message-ID: <YD0lBD9t3Od0dgXp@decadent.org.uk>
+Content-Disposition: inline
+
+From: Ben Hutchings <ben@decadent.org.uk>
+
+From: Peter Zijlstra <peterz@infradead.org>
+
+commit c74aef2d06a9f59cece89093eecc552933cba72a upstream.
+
+There was a reported suspicion about a race between exit_pi_state_list()
+and put_pi_state(). The same report mentioned the comment with
+put_pi_state() said it should be called with hb->lock held, and it no
+longer is in all places.
+
+As it turns out, the pi_state->owner serialization is indeed broken. As per
+the new rules:
+
+ 734009e96d19 ("futex: Change locking rules")
+
+pi_state->owner should be serialized by pi_state->pi_mutex.wait_lock.
+For the sites setting pi_state->owner we already hold wait_lock (where
+required) but exit_pi_state_list() and put_pi_state() were not and
+raced on clearing it.
+
+Fixes: 734009e96d19 ("futex: Change locking rules")
+Reported-by: Gratian Crisan <gratian.crisan@ni.com>
+Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
+Cc: dvhart@infradead.org
+Cc: stable@vger.kernel.org
+Link:
+https://lkml.kernel.org/r/20170922154806.jd3ffltfk24m4o4y@hirez.programming.kicks-ass.net
+[bwh: Backported to 4.9: adjust context]
+Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ kernel/futex.c | 21 ++++++++++++++-------
+ 1 file changed, 14 insertions(+), 7 deletions(-)
+
+--- a/kernel/futex.c
++++ b/kernel/futex.c
+@@ -868,8 +868,6 @@ static void get_pi_state(struct futex_pi
+ /*
+ * Drops a reference to the pi_state object and frees or caches it
+ * when the last reference is gone.
+- *
+- * Must be called with the hb lock held.
+ */
+ static void put_pi_state(struct futex_pi_state *pi_state)
+ {
+@@ -884,13 +882,15 @@ static void put_pi_state(struct futex_pi
+ * and has cleaned up the pi_state already
+ */
+ if (pi_state->owner) {
++ raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
+ pi_state_update_owner(pi_state, NULL);
+ rt_mutex_proxy_unlock(&pi_state->pi_mutex);
++ raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
+ }
+
+- if (current->pi_state_cache)
++ if (current->pi_state_cache) {
+ kfree(pi_state);
+- else {
++ } else {
+ /*
+ * pi_state->list is already empty.
+ * clear pi_state->owner.
+@@ -949,13 +949,14 @@ static void exit_pi_state_list(struct ta
+ raw_spin_unlock_irq(&curr->pi_lock);
+
+ spin_lock(&hb->lock);
+-
+- raw_spin_lock_irq(&curr->pi_lock);
++ raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
++ raw_spin_lock(&curr->pi_lock);
+ /*
+ * We dropped the pi-lock, so re-check whether this
+ * task still owns the PI-state:
+ */
+ if (head->next != next) {
++ raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
+ spin_unlock(&hb->lock);
+ continue;
+ }
+@@ -964,9 +965,10 @@ static void exit_pi_state_list(struct ta
+ WARN_ON(list_empty(&pi_state->list));
+ list_del_init(&pi_state->list);
+ pi_state->owner = NULL;
+- raw_spin_unlock_irq(&curr->pi_lock);
++ raw_spin_unlock(&curr->pi_lock);
+
+ get_pi_state(pi_state);
++ raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
+ spin_unlock(&hb->lock);
+
+ rt_mutex_futex_unlock(&pi_state->pi_mutex);
+@@ -1349,6 +1351,10 @@ static int attach_to_pi_owner(u32 __user
+
+ WARN_ON(!list_empty(&pi_state->list));
+ list_add(&pi_state->list, &p->pi_state_list);
++ /*
++ * Assignment without holding pi_state->pi_mutex.wait_lock is safe
++ * because there is no concurrency as the object is not published yet.
++ */
+ pi_state->owner = p;
+ raw_spin_unlock_irq(&p->pi_lock);
+
+@@ -3027,6 +3033,7 @@ retry:
+ raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
+ spin_unlock(&hb->lock);
+
++ /* drops pi_state->pi_mutex.wait_lock */
+ ret = wake_futex_pi(uaddr, uval, pi_state);
+
+ put_pi_state(pi_state);
--- /dev/null
+From foo@baz Thu Mar 4 02:09:29 PM CET 2021
+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Mon, 1 Mar 2021 18:31:39 +0100
+Subject: futex: Futex_unlock_pi() determinism
+To: stable@vger.kernel.org
+Cc: Lee Jones <lee.jones@linaro.org>, "Luis Claudio R. Goncalves" <lgoncalv@redhat.com>
+Message-ID: <YD0k+9Ukc5MhhU8V@decadent.org.uk>
+Content-Disposition: inline
+
+From: Ben Hutchings <ben@decadent.org.uk>
+
+From: Peter Zijlstra <peterz@infradead.org>
+
+commit bebe5b514345f09be2c15e414d076b02ecb9cce8 upstream.
+
+The problem with returning -EAGAIN when the waiter state mismatches is that
+it becomes very hard to proof a bounded execution time on the
+operation. And seeing that this is a RT operation, this is somewhat
+important.
+
+While in practise; given the previous patch; it will be very unlikely to
+ever really take more than one or two rounds, proving so becomes rather
+hard.
+
+However, now that modifying wait_list is done while holding both hb->lock
+and wait_lock, the scenario can be avoided entirely by acquiring wait_lock
+while still holding hb-lock. Doing a hand-over, without leaving a hole.
+
+Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Cc: juri.lelli@arm.com
+Cc: bigeasy@linutronix.de
+Cc: xlpang@redhat.com
+Cc: rostedt@goodmis.org
+Cc: mathieu.desnoyers@efficios.com
+Cc: jdesfossez@efficios.com
+Cc: dvhart@infradead.org
+Cc: bristot@redhat.com
+Link: http://lkml.kernel.org/r/20170322104152.112378812@infradead.org
+Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
+Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ kernel/futex.c | 24 +++++++++++-------------
+ 1 file changed, 11 insertions(+), 13 deletions(-)
+
+--- a/kernel/futex.c
++++ b/kernel/futex.c
+@@ -1555,15 +1555,10 @@ static int wake_futex_pi(u32 __user *uad
+ WAKE_Q(wake_q);
+ int ret = 0;
+
+- raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
+ new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
+- if (!new_owner) {
++ if (WARN_ON_ONCE(!new_owner)) {
+ /*
+- * Since we held neither hb->lock nor wait_lock when coming
+- * into this function, we could have raced with futex_lock_pi()
+- * such that we might observe @this futex_q waiter, but the
+- * rt_mutex's wait_list can be empty (either still, or again,
+- * depending on which side we land).
++ * As per the comment in futex_unlock_pi() this should not happen.
+ *
+ * When this happens, give up our locks and try again, giving
+ * the futex_lock_pi() instance time to complete, either by
+@@ -3018,15 +3013,18 @@ retry:
+ if (pi_state->owner != current)
+ goto out_unlock;
+
++ get_pi_state(pi_state);
+ /*
+- * Grab a reference on the pi_state and drop hb->lock.
++ * Since modifying the wait_list is done while holding both
++ * hb->lock and wait_lock, holding either is sufficient to
++ * observe it.
+ *
+- * The reference ensures pi_state lives, dropping the hb->lock
+- * is tricky.. wake_futex_pi() will take rt_mutex::wait_lock to
+- * close the races against futex_lock_pi(), but in case of
+- * _any_ fail we'll abort and retry the whole deal.
++ * By taking wait_lock while still holding hb->lock, we ensure
++ * there is no point where we hold neither; and therefore
++ * wake_futex_pi() must observe a state consistent with what we
++ * observed.
+ */
+- get_pi_state(pi_state);
++ raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
+ spin_unlock(&hb->lock);
+
+ ret = wake_futex_pi(uaddr, uval, pi_state);
--- /dev/null
+From foo@baz Thu Mar 4 02:09:29 PM CET 2021
+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Mon, 1 Mar 2021 18:31:30 +0100
+Subject: futex: Pull rt_mutex_futex_unlock() out from under hb->lock
+To: stable@vger.kernel.org
+Cc: Lee Jones <lee.jones@linaro.org>, "Luis Claudio R. Goncalves" <lgoncalv@redhat.com>
+Message-ID: <YD0k8hpgdxeBVa2M@decadent.org.uk>
+Content-Disposition: inline
+
+From: Ben Hutchings <ben@decadent.org.uk>
+
+From: Peter Zijlstra <peterz@infradead.org>
+
+commit 16ffa12d742534d4ff73e8b3a4e81c1de39196f0 upstream.
+
+There's a number of 'interesting' problems, all caused by holding
+hb->lock while doing the rt_mutex_unlock() equivalient.
+
+Notably:
+
+ - a PI inversion on hb->lock; and,
+
+ - a SCHED_DEADLINE crash because of pointer instability.
+
+The previous changes:
+
+ - changed the locking rules to cover {uval,pi_state} with wait_lock.
+
+ - allow to do rt_mutex_futex_unlock() without dropping wait_lock; which in
+ turn allows to rely on wait_lock atomicity completely.
+
+ - simplified the waiter conundrum.
+
+It's now sufficient to hold rtmutex::wait_lock and a reference on the
+pi_state to protect the state consistency, so hb->lock can be dropped
+before calling rt_mutex_futex_unlock().
+
+Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Cc: juri.lelli@arm.com
+Cc: bigeasy@linutronix.de
+Cc: xlpang@redhat.com
+Cc: rostedt@goodmis.org
+Cc: mathieu.desnoyers@efficios.com
+Cc: jdesfossez@efficios.com
+Cc: dvhart@infradead.org
+Cc: bristot@redhat.com
+Link: http://lkml.kernel.org/r/20170322104151.900002056@infradead.org
+Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
+[bwh: Backported to 4.9: adjust context]
+Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ kernel/futex.c | 111 ++++++++++++++++++++++++++++++++++-----------------------
+ 1 file changed, 68 insertions(+), 43 deletions(-)
+
+--- a/kernel/futex.c
++++ b/kernel/futex.c
+@@ -966,10 +966,12 @@ static void exit_pi_state_list(struct ta
+ pi_state->owner = NULL;
+ raw_spin_unlock_irq(&curr->pi_lock);
+
+- rt_mutex_futex_unlock(&pi_state->pi_mutex);
+-
++ get_pi_state(pi_state);
+ spin_unlock(&hb->lock);
+
++ rt_mutex_futex_unlock(&pi_state->pi_mutex);
++ put_pi_state(pi_state);
++
+ raw_spin_lock_irq(&curr->pi_lock);
+ }
+ raw_spin_unlock_irq(&curr->pi_lock);
+@@ -1083,6 +1085,11 @@ static int attach_to_pi_state(u32 __user
+ * has dropped the hb->lock in between queue_me() and unqueue_me_pi(),
+ * which in turn means that futex_lock_pi() still has a reference on
+ * our pi_state.
++ *
++ * The waiter holding a reference on @pi_state also protects against
++ * the unlocked put_pi_state() in futex_unlock_pi(), futex_lock_pi()
++ * and futex_wait_requeue_pi() as it cannot go to 0 and consequently
++ * free pi_state before we can take a reference ourselves.
+ */
+ WARN_ON(!atomic_read(&pi_state->refcount));
+
+@@ -1537,48 +1544,40 @@ static void mark_wake_futex(struct wake_
+ q->lock_ptr = NULL;
+ }
+
+-static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *top_waiter,
+- struct futex_hash_bucket *hb)
++/*
++ * Caller must hold a reference on @pi_state.
++ */
++static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)
+ {
+- struct task_struct *new_owner;
+- struct futex_pi_state *pi_state = top_waiter->pi_state;
+ u32 uninitialized_var(curval), newval;
++ struct task_struct *new_owner;
++ bool deboost = false;
+ WAKE_Q(wake_q);
+- bool deboost;
+ int ret = 0;
+
+- if (!pi_state)
+- return -EINVAL;
+-
+- /*
+- * If current does not own the pi_state then the futex is
+- * inconsistent and user space fiddled with the futex value.
+- */
+- if (pi_state->owner != current)
+- return -EINVAL;
+-
+ raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
+ new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
+-
+- /*
+- * When we interleave with futex_lock_pi() where it does
+- * rt_mutex_timed_futex_lock(), we might observe @top_waiter futex_q waiter,
+- * but the rt_mutex's wait_list can be empty (either still, or again,
+- * depending on which side we land).
+- *
+- * When this happens, give up our locks and try again, giving the
+- * futex_lock_pi() instance time to complete, either by waiting on the
+- * rtmutex or removing itself from the futex queue.
+- */
+ if (!new_owner) {
+- raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
+- return -EAGAIN;
++ /*
++ * Since we held neither hb->lock nor wait_lock when coming
++ * into this function, we could have raced with futex_lock_pi()
++ * such that we might observe @this futex_q waiter, but the
++ * rt_mutex's wait_list can be empty (either still, or again,
++ * depending on which side we land).
++ *
++ * When this happens, give up our locks and try again, giving
++ * the futex_lock_pi() instance time to complete, either by
++ * waiting on the rtmutex or removing itself from the futex
++ * queue.
++ */
++ ret = -EAGAIN;
++ goto out_unlock;
+ }
+
+ /*
+- * We pass it to the next owner. The WAITERS bit is always
+- * kept enabled while there is PI state around. We cleanup the
+- * owner died bit, because we are the owner.
++ * We pass it to the next owner. The WAITERS bit is always kept
++ * enabled while there is PI state around. We cleanup the owner
++ * died bit, because we are the owner.
+ */
+ newval = FUTEX_WAITERS | task_pid_vnr(new_owner);
+
+@@ -1611,15 +1610,15 @@ static int wake_futex_pi(u32 __user *uad
+ deboost = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
+ }
+
++out_unlock:
+ raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
+- spin_unlock(&hb->lock);
+
+ if (deboost) {
+ wake_up_q(&wake_q);
+ rt_mutex_adjust_prio(current);
+ }
+
+- return 0;
++ return ret;
+ }
+
+ /*
+@@ -2493,7 +2492,7 @@ retry:
+ if (get_futex_value_locked(&uval, uaddr))
+ goto handle_fault;
+
+- while (1) {
++ for (;;) {
+ newval = (uval & FUTEX_OWNER_DIED) | newtid;
+
+ if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))
+@@ -3006,10 +3005,36 @@ retry:
+ */
+ top_waiter = futex_top_waiter(hb, &key);
+ if (top_waiter) {
+- ret = wake_futex_pi(uaddr, uval, top_waiter, hb);
++ struct futex_pi_state *pi_state = top_waiter->pi_state;
++
++ ret = -EINVAL;
++ if (!pi_state)
++ goto out_unlock;
++
+ /*
+- * In case of success wake_futex_pi dropped the hash
+- * bucket lock.
++ * If current does not own the pi_state then the futex is
++ * inconsistent and user space fiddled with the futex value.
++ */
++ if (pi_state->owner != current)
++ goto out_unlock;
++
++ /*
++ * Grab a reference on the pi_state and drop hb->lock.
++ *
++ * The reference ensures pi_state lives, dropping the hb->lock
++ * is tricky.. wake_futex_pi() will take rt_mutex::wait_lock to
++ * close the races against futex_lock_pi(), but in case of
++ * _any_ fail we'll abort and retry the whole deal.
++ */
++ get_pi_state(pi_state);
++ spin_unlock(&hb->lock);
++
++ ret = wake_futex_pi(uaddr, uval, pi_state);
++
++ put_pi_state(pi_state);
++
++ /*
++ * Success, we're done! No tricky corner cases.
+ */
+ if (!ret)
+ goto out_putkey;
+@@ -3024,7 +3049,6 @@ retry:
+ * setting the FUTEX_WAITERS bit. Try again.
+ */
+ if (ret == -EAGAIN) {
+- spin_unlock(&hb->lock);
+ put_futex_key(&key);
+ goto retry;
+ }
+@@ -3032,7 +3056,7 @@ retry:
+ * wake_futex_pi has detected invalid state. Tell user
+ * space.
+ */
+- goto out_unlock;
++ goto out_putkey;
+ }
+
+ /*
+@@ -3042,8 +3066,10 @@ retry:
+ * preserve the WAITERS bit not the OWNER_DIED one. We are the
+ * owner.
+ */
+- if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))
++ if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0)) {
++ spin_unlock(&hb->lock);
+ goto pi_faulted;
++ }
+
+ /*
+ * If uval has changed, let user space handle it.
+@@ -3057,7 +3083,6 @@ out_putkey:
+ return ret;
+
+ pi_faulted:
+- spin_unlock(&hb->lock);
+ put_futex_key(&key);
+
+ ret = fault_in_user_writeable(uaddr);
projects / thirdparty / kernel / stable-queue.git / commitdiff
