1 // SPDX-License-Identifier: GPL-2.0
3 * Asynchronous refcounty things
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
9 #include <linux/closure.h>
10 #include <linux/debugfs.h>
11 #include <linux/export.h>
12 #include <linux/rcupdate.h>
13 #include <linux/seq_file.h>
14 #include <linux/sched/debug.h>
16 static inline void closure_put_after_sub(struct closure
*cl
, int flags
)
18 int r
= flags
& CLOSURE_REMAINING_MASK
;
20 BUG_ON(flags
& CLOSURE_GUARD_MASK
);
21 BUG_ON(!r
&& (flags
& ~CLOSURE_DESTRUCTOR
));
24 smp_acquire__after_ctrl_dep();
26 cl
->closure_get_happened
= false;
28 if (cl
->fn
&& !(flags
& CLOSURE_DESTRUCTOR
)) {
29 atomic_set(&cl
->remaining
,
30 CLOSURE_REMAINING_INITIALIZER
);
33 struct closure
*parent
= cl
->parent
;
34 closure_fn
*destructor
= cl
->fn
;
36 closure_debug_destroy(cl
);
39 destructor(&cl
->work
);
47 /* For clearing flags with the same atomic op as a put */
48 void closure_sub(struct closure
*cl
, int v
)
50 closure_put_after_sub(cl
, atomic_sub_return_release(v
, &cl
->remaining
));
52 EXPORT_SYMBOL(closure_sub
);
55 * closure_put - decrement a closure's refcount
57 void closure_put(struct closure
*cl
)
59 closure_put_after_sub(cl
, atomic_dec_return_release(&cl
->remaining
));
61 EXPORT_SYMBOL(closure_put
);
64 * closure_wake_up - wake up all closures on a wait list, without memory barrier
66 void __closure_wake_up(struct closure_waitlist
*wait_list
)
68 struct llist_node
*list
;
69 struct closure
*cl
, *t
;
70 struct llist_node
*reverse
= NULL
;
72 list
= llist_del_all(&wait_list
->list
);
74 /* We first reverse the list to preserve FIFO ordering and fairness */
75 reverse
= llist_reverse_order(list
);
77 /* Then do the wakeups */
78 llist_for_each_entry_safe(cl
, t
, reverse
, list
) {
79 closure_set_waiting(cl
, 0);
80 closure_sub(cl
, CLOSURE_WAITING
+ 1);
83 EXPORT_SYMBOL(__closure_wake_up
);
86 * closure_wait - add a closure to a waitlist
87 * @waitlist: will own a ref on @cl, which will be released when
88 * closure_wake_up() is called on @waitlist.
89 * @cl: closure pointer.
92 bool closure_wait(struct closure_waitlist
*waitlist
, struct closure
*cl
)
94 if (atomic_read(&cl
->remaining
) & CLOSURE_WAITING
)
97 cl
->closure_get_happened
= true;
98 closure_set_waiting(cl
, _RET_IP_
);
99 atomic_add(CLOSURE_WAITING
+ 1, &cl
->remaining
);
100 llist_add(&cl
->list
, &waitlist
->list
);
104 EXPORT_SYMBOL(closure_wait
);
106 struct closure_syncer
{
107 struct task_struct
*task
;
111 static CLOSURE_CALLBACK(closure_sync_fn
)
113 struct closure
*cl
= container_of(ws
, struct closure
, work
);
114 struct closure_syncer
*s
= cl
->s
;
115 struct task_struct
*p
;
118 p
= READ_ONCE(s
->task
);
124 void __sched
__closure_sync(struct closure
*cl
)
126 struct closure_syncer s
= { .task
= current
};
129 continue_at(cl
, closure_sync_fn
, NULL
);
132 set_current_state(TASK_UNINTERRUPTIBLE
);
138 __set_current_state(TASK_RUNNING
);
140 EXPORT_SYMBOL(__closure_sync
);
142 #ifdef CONFIG_DEBUG_CLOSURES
144 static LIST_HEAD(closure_list
);
145 static DEFINE_SPINLOCK(closure_list_lock
);
147 void closure_debug_create(struct closure
*cl
)
151 BUG_ON(cl
->magic
== CLOSURE_MAGIC_ALIVE
);
152 cl
->magic
= CLOSURE_MAGIC_ALIVE
;
154 spin_lock_irqsave(&closure_list_lock
, flags
);
155 list_add(&cl
->all
, &closure_list
);
156 spin_unlock_irqrestore(&closure_list_lock
, flags
);
158 EXPORT_SYMBOL(closure_debug_create
);
160 void closure_debug_destroy(struct closure
*cl
)
164 BUG_ON(cl
->magic
!= CLOSURE_MAGIC_ALIVE
);
165 cl
->magic
= CLOSURE_MAGIC_DEAD
;
167 spin_lock_irqsave(&closure_list_lock
, flags
);
169 spin_unlock_irqrestore(&closure_list_lock
, flags
);
171 EXPORT_SYMBOL(closure_debug_destroy
);
173 static int debug_show(struct seq_file
*f
, void *data
)
177 spin_lock_irq(&closure_list_lock
);
179 list_for_each_entry(cl
, &closure_list
, all
) {
180 int r
= atomic_read(&cl
->remaining
);
182 seq_printf(f
, "%p: %pS -> %pS p %p r %i ",
183 cl
, (void *) cl
->ip
, cl
->fn
, cl
->parent
,
184 r
& CLOSURE_REMAINING_MASK
);
186 seq_printf(f
, "%s%s\n",
187 test_bit(WORK_STRUCT_PENDING_BIT
,
188 work_data_bits(&cl
->work
)) ? "Q" : "",
189 r
& CLOSURE_RUNNING
? "R" : "");
191 if (r
& CLOSURE_WAITING
)
192 seq_printf(f
, " W %pS\n",
193 (void *) cl
->waiting_on
);
198 spin_unlock_irq(&closure_list_lock
);
202 DEFINE_SHOW_ATTRIBUTE(debug
);
204 static int __init
closure_debug_init(void)
206 debugfs_create_file("closures", 0400, NULL
, NULL
, &debug_fops
);
209 late_initcall(closure_debug_init
)