unwind: Make unwind_task_info::unwind_mask consistent

The unwind_task_info::unwind_mask was manipulated using a mixture of:

  regular store
  WRITE_ONCE()
  try_cmpxchg()
  set_bit()
  atomic_long_*()

Clean up and make it consistently atomic_long_t.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20250924080119.384384486@infradead.org

diff --git a/include/linux/unwind_deferred.h b/include/linux/unwind_deferred.h
index 196e12c1449eafb6b81ce330c2ae0a113978d156..f4743c8cff4c2f350663b620c4a4688e6817a464 100644 (file)
--- a/include/linux/unwind_deferred.h
+++ b/include/linux/unwind_deferred.h
@@ -46,7 +46,7 @@ void unwind_deferred_task_exit(struct task_struct *task);
 static __always_inline void unwind_reset_info(void)
 {
        struct unwind_task_info *info = &current->unwind_info;
-       unsigned long bits = info->unwind_mask;
+       unsigned long bits = atomic_long_read(&info->unwind_mask);
 
        /* Was there any unwinding? */
        if (likely(!bits))
@@ -56,7 +56,7 @@ static __always_inline void unwind_reset_info(void)
                /* Is a task_work going to run again before going back */
                if (bits & UNWIND_PENDING)
                        return;
-       } while (!try_cmpxchg(&info->unwind_mask, &bits, 0UL));
+       } while (!atomic_long_try_cmpxchg(&info->unwind_mask, &bits, 0UL));
        current->unwind_info.id.id = 0;
 
        if (unlikely(info->cache)) {

diff --git a/include/linux/unwind_deferred_types.h b/include/linux/unwind_deferred_types.h
index 29452ff4985971316394fd2266505da49b2207b7..0a4c8ddbbc57d4e7a5700f14250193be5c92c7f8 100644 (file)
--- a/include/linux/unwind_deferred_types.h
+++ b/include/linux/unwind_deferred_types.h
@@ -3,6 +3,7 @@
 #define _LINUX_UNWIND_USER_DEFERRED_TYPES_H
 
 #include <linux/types.h>
+#include <linux/atomic.h>
 
 struct unwind_cache {
        unsigned long           unwind_completed;
@@ -32,7 +33,7 @@ union unwind_task_id {
 };
 
 struct unwind_task_info {
-       unsigned long           unwind_mask;
+       atomic_long_t           unwind_mask;
        struct unwind_cache     *cache;
        struct callback_head    work;
        union unwind_task_id    id;

diff --git a/kernel/unwind/deferred.c b/kernel/unwind/deferred.c
index 09617d8ae24bca662ba2dc3053e05e84937e0ecc..a88fb481c4a3fc9d1b9e97350640e9acf3513e86 100644 (file)
--- a/kernel/unwind/deferred.c
+++ b/kernel/unwind/deferred.c
@@ -53,7 +53,7 @@ DEFINE_STATIC_SRCU(unwind_srcu);
 
 static inline bool unwind_pending(struct unwind_task_info *info)
 {
-       return test_bit(UNWIND_PENDING_BIT, &info->unwind_mask);
+       return atomic_long_read(&info->unwind_mask) & UNWIND_PENDING;
 }
 
 /*
@@ -141,7 +141,7 @@ int unwind_user_faultable(struct unwind_stacktrace *trace)
        cache->nr_entries = trace->nr;
 
        /* Clear nr_entries on way back to user space */
-       set_bit(UNWIND_USED_BIT, &info->unwind_mask);
+       atomic_long_or(UNWIND_USED, &info->unwind_mask);
 
        return 0;
 }
@@ -159,7 +159,7 @@ static void process_unwind_deferred(struct task_struct *task)
 
        /* Clear pending bit but make sure to have the current bits */
        bits = atomic_long_fetch_andnot(UNWIND_PENDING,
-                                 (atomic_long_t *)&info->unwind_mask);
+                                       &info->unwind_mask);
        /*
         * From here on out, the callback must always be called, even if it's
         * just an empty trace.
@@ -264,7 +264,7 @@ int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
 
        *cookie = get_cookie(info);
 
-       old = READ_ONCE(info->unwind_mask);
+       old = atomic_long_read(&info->unwind_mask);
 
        /* Is this already queued or executed */
        if (old & bit)
@@ -277,7 +277,7 @@ int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
         * to have a callback.
         */
        bits = UNWIND_PENDING | bit;
-       old = atomic_long_fetch_or(bits, (atomic_long_t *)&info->unwind_mask);
+       old = atomic_long_fetch_or(bits, &info->unwind_mask);
        if (old & bits) {
                /*
                 * If the work's bit was set, whatever set it had better
@@ -291,7 +291,7 @@ int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
        ret = task_work_add(current, &info->work, twa_mode);
 
        if (WARN_ON_ONCE(ret))
-               WRITE_ONCE(info->unwind_mask, 0);
+               atomic_long_set(&info->unwind_mask, 0);
 
        return ret;
 }
@@ -323,7 +323,8 @@ void unwind_deferred_cancel(struct unwind_work *work)
        guard(rcu)();
        /* Clear this bit from all threads */
        for_each_process_thread(g, t) {
-               clear_bit(bit, &t->unwind_info.unwind_mask);
+               atomic_long_andnot(BIT(bit),
+                                  &t->unwind_info.unwind_mask);
                if (t->unwind_info.cache)
                        clear_bit(bit, &t->unwind_info.cache->unwind_completed);
        }
@@ -353,7 +354,7 @@ void unwind_task_init(struct task_struct *task)
 
        memset(info, 0, sizeof(*info));
        init_task_work(&info->work, unwind_deferred_task_work);
-       info->unwind_mask = 0;
+       atomic_long_set(&info->unwind_mask, 0);
 }
 
 void unwind_task_free(struct task_struct *task)