]>
git.ipfire.org Git - people/ms/linux.git/blob - fs/io-wq.h
1 #ifndef INTERNAL_IO_WQ_H
2 #define INTERNAL_IO_WQ_H
4 #include <linux/refcount.h>
10 IO_WQ_WORK_HASHED
= 2,
11 IO_WQ_WORK_UNBOUND
= 4,
12 IO_WQ_WORK_CONCURRENT
= 16,
14 IO_WQ_HASH_SHIFT
= 24, /* upper 8 bits are used for hash key */
18 IO_WQ_CANCEL_OK
, /* cancelled before started */
19 IO_WQ_CANCEL_RUNNING
, /* found, running, and attempted cancelled */
20 IO_WQ_CANCEL_NOTFOUND
, /* work not found */
23 struct io_wq_work_node
{
24 struct io_wq_work_node
*next
;
27 struct io_wq_work_list
{
28 struct io_wq_work_node
*first
;
29 struct io_wq_work_node
*last
;
32 #define wq_list_for_each(pos, prv, head) \
33 for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
35 #define wq_list_for_each_resume(pos, prv) \
36 for (; pos; prv = pos, pos = (pos)->next)
38 #define wq_list_empty(list) (READ_ONCE((list)->first) == NULL)
39 #define INIT_WQ_LIST(list) do { \
40 (list)->first = NULL; \
43 static inline void wq_list_add_after(struct io_wq_work_node
*node
,
44 struct io_wq_work_node
*pos
,
45 struct io_wq_work_list
*list
)
47 struct io_wq_work_node
*next
= pos
->next
;
56 * wq_list_merge - merge the second list to the first one.
57 * @list0: the first list
58 * @list1: the second list
59 * Return the first node after mergence.
61 static inline struct io_wq_work_node
*wq_list_merge(struct io_wq_work_list
*list0
,
62 struct io_wq_work_list
*list1
)
64 struct io_wq_work_node
*ret
;
70 list0
->last
->next
= list1
->first
;
77 static inline void wq_list_add_tail(struct io_wq_work_node
*node
,
78 struct io_wq_work_list
*list
)
83 WRITE_ONCE(list
->first
, node
);
85 list
->last
->next
= node
;
90 static inline void wq_list_add_head(struct io_wq_work_node
*node
,
91 struct io_wq_work_list
*list
)
93 node
->next
= list
->first
;
96 WRITE_ONCE(list
->first
, node
);
99 static inline void wq_list_cut(struct io_wq_work_list
*list
,
100 struct io_wq_work_node
*last
,
101 struct io_wq_work_node
*prev
)
103 /* first in the list, if prev==NULL */
105 WRITE_ONCE(list
->first
, last
->next
);
107 prev
->next
= last
->next
;
109 if (last
== list
->last
)
114 static inline void __wq_list_splice(struct io_wq_work_list
*list
,
115 struct io_wq_work_node
*to
)
117 list
->last
->next
= to
->next
;
118 to
->next
= list
->first
;
122 static inline bool wq_list_splice(struct io_wq_work_list
*list
,
123 struct io_wq_work_node
*to
)
125 if (!wq_list_empty(list
)) {
126 __wq_list_splice(list
, to
);
132 static inline void wq_stack_add_head(struct io_wq_work_node
*node
,
133 struct io_wq_work_node
*stack
)
135 node
->next
= stack
->next
;
139 static inline void wq_list_del(struct io_wq_work_list
*list
,
140 struct io_wq_work_node
*node
,
141 struct io_wq_work_node
*prev
)
143 wq_list_cut(list
, node
, prev
);
147 struct io_wq_work_node
*wq_stack_extract(struct io_wq_work_node
*stack
)
149 struct io_wq_work_node
*node
= stack
->next
;
151 stack
->next
= node
->next
;
156 struct io_wq_work_node list
;
161 static inline struct io_wq_work
*wq_next_work(struct io_wq_work
*work
)
163 if (!work
->list
.next
)
166 return container_of(work
->list
.next
, struct io_wq_work
, list
);
169 typedef struct io_wq_work
*(free_work_fn
)(struct io_wq_work
*);
170 typedef void (io_wq_work_fn
)(struct io_wq_work
*);
175 struct wait_queue_head wait
;
178 static inline void io_wq_put_hash(struct io_wq_hash
*hash
)
180 if (refcount_dec_and_test(&hash
->refs
))
185 struct io_wq_hash
*hash
;
186 struct task_struct
*task
;
187 io_wq_work_fn
*do_work
;
188 free_work_fn
*free_work
;
191 struct io_wq
*io_wq_create(unsigned bounded
, struct io_wq_data
*data
);
192 void io_wq_exit_start(struct io_wq
*wq
);
193 void io_wq_put_and_exit(struct io_wq
*wq
);
195 void io_wq_enqueue(struct io_wq
*wq
, struct io_wq_work
*work
);
196 void io_wq_hash_work(struct io_wq_work
*work
, void *val
);
198 int io_wq_cpu_affinity(struct io_wq
*wq
, cpumask_var_t mask
);
199 int io_wq_max_workers(struct io_wq
*wq
, int *new_count
);
201 static inline bool io_wq_is_hashed(struct io_wq_work
*work
)
203 return work
->flags
& IO_WQ_WORK_HASHED
;
206 typedef bool (work_cancel_fn
)(struct io_wq_work
*, void *);
208 enum io_wq_cancel
io_wq_cancel_cb(struct io_wq
*wq
, work_cancel_fn
*cancel
,
209 void *data
, bool cancel_all
);
211 #if defined(CONFIG_IO_WQ)
212 extern void io_wq_worker_sleeping(struct task_struct
*);
213 extern void io_wq_worker_running(struct task_struct
*);
215 static inline void io_wq_worker_sleeping(struct task_struct
*tsk
)
218 static inline void io_wq_worker_running(struct task_struct
*tsk
)
223 static inline bool io_wq_current_is_worker(void)
225 return in_task() && (current
->flags
& PF_IO_WORKER
) &&
226 current
->worker_private
;