[people/ms/linux.git] / fs / io-wq.h

#ifndef INTERNAL_IO_WQ_H
#define INTERNAL_IO_WQ_H

#include <linux/refcount.h>

struct io_wq;

enum {
	IO_WQ_WORK_CANCEL	= 1,
	IO_WQ_WORK_HASHED	= 2,
	IO_WQ_WORK_UNBOUND	= 4,
	IO_WQ_WORK_CONCURRENT	= 16,

	IO_WQ_HASH_SHIFT	= 24,	/* upper 8 bits are used for hash key */
};

enum io_wq_cancel {
	IO_WQ_CANCEL_OK,	/* cancelled before started */
	IO_WQ_CANCEL_RUNNING,	/* found, running, and attempted cancelled */
	IO_WQ_CANCEL_NOTFOUND,	/* work not found */
};

struct io_wq_work_node {
	struct io_wq_work_node *next;
};

struct io_wq_work_list {
	struct io_wq_work_node *first;
	struct io_wq_work_node *last;
};

#define wq_list_for_each(pos, prv, head)			\
	for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)

#define wq_list_for_each_resume(pos, prv)			\
	for (; pos; prv = pos, pos = (pos)->next)

#define wq_list_empty(list)	(READ_ONCE((list)->first) == NULL)
#define INIT_WQ_LIST(list)	do {				\
	(list)->first = NULL;					\
} while (0)

static inline void wq_list_add_after(struct io_wq_work_node *node,
				     struct io_wq_work_node *pos,
				     struct io_wq_work_list *list)
{
	struct io_wq_work_node *next = pos->next;

	pos->next = node;
	node->next = next;
	if (!next)
		list->last = node;
}

/**
 * wq_list_merge - merge the second list to the first one.
 * @list0: the first list
 * @list1: the second list
 * Return the first node after mergence.
 */
static inline struct io_wq_work_node *wq_list_merge(struct io_wq_work_list *list0,
						    struct io_wq_work_list *list1)
{
	struct io_wq_work_node *ret;

	if (!list0->first) {
		ret = list1->first;
	} else {
		ret = list0->first;
		list0->last->next = list1->first;
	}
	INIT_WQ_LIST(list0);
	INIT_WQ_LIST(list1);
	return ret;
}

static inline void wq_list_add_tail(struct io_wq_work_node *node,
				    struct io_wq_work_list *list)
{
	node->next = NULL;
	if (!list->first) {
		list->last = node;
		WRITE_ONCE(list->first, node);
	} else {
		list->last->next = node;
		list->last = node;
	}
}

static inline void wq_list_add_head(struct io_wq_work_node *node,
				    struct io_wq_work_list *list)
{
	node->next = list->first;
	if (!node->next)
		list->last = node;
	WRITE_ONCE(list->first, node);
}

static inline void wq_list_cut(struct io_wq_work_list *list,
			       struct io_wq_work_node *last,
			       struct io_wq_work_node *prev)
{
	/* first in the list, if prev==NULL */
	if (!prev)
		WRITE_ONCE(list->first, last->next);
	else
		prev->next = last->next;

	if (last == list->last)
		list->last = prev;
	last->next = NULL;
}

static inline void __wq_list_splice(struct io_wq_work_list *list,
				    struct io_wq_work_node *to)
{
	list->last->next = to->next;
	to->next = list->first;
	INIT_WQ_LIST(list);
}

static inline bool wq_list_splice(struct io_wq_work_list *list,
				  struct io_wq_work_node *to)
{
	if (!wq_list_empty(list)) {
		__wq_list_splice(list, to);
		return true;
	}
	return false;
}

static inline void wq_stack_add_head(struct io_wq_work_node *node,
				     struct io_wq_work_node *stack)
{
	node->next = stack->next;
	stack->next = node;
}

static inline void wq_list_del(struct io_wq_work_list *list,
			       struct io_wq_work_node *node,
			       struct io_wq_work_node *prev)
{
	wq_list_cut(list, node, prev);
}

static inline
struct io_wq_work_node *wq_stack_extract(struct io_wq_work_node *stack)
{
	struct io_wq_work_node *node = stack->next;

	stack->next = node->next;
	return node;
}

struct io_wq_work {
	struct io_wq_work_node list;
	unsigned flags;
};

static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
{
	if (!work->list.next)
		return NULL;

	return container_of(work->list.next, struct io_wq_work, list);
}

typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *);
typedef void (io_wq_work_fn)(struct io_wq_work *);

struct io_wq_hash {
	refcount_t refs;
	unsigned long map;
	struct wait_queue_head wait;
};

static inline void io_wq_put_hash(struct io_wq_hash *hash)
{
	if (refcount_dec_and_test(&hash->refs))
		kfree(hash);
}

struct io_wq_data {
	struct io_wq_hash *hash;
	struct task_struct *task;
	io_wq_work_fn *do_work;
	free_work_fn *free_work;
};

struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);
void io_wq_exit_start(struct io_wq *wq);
void io_wq_put_and_exit(struct io_wq *wq);

void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);
void io_wq_hash_work(struct io_wq_work *work, void *val);

int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask);
int io_wq_max_workers(struct io_wq *wq, int *new_count);

static inline bool io_wq_is_hashed(struct io_wq_work *work)
{
	return work->flags & IO_WQ_WORK_HASHED;
}

typedef bool (work_cancel_fn)(struct io_wq_work *, void *);

enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
					void *data, bool cancel_all);

#if defined(CONFIG_IO_WQ)
extern void io_wq_worker_sleeping(struct task_struct *);
extern void io_wq_worker_running(struct task_struct *);
#else
static inline void io_wq_worker_sleeping(struct task_struct *tsk)
{
}
static inline void io_wq_worker_running(struct task_struct *tsk)
{
}
#endif

static inline bool io_wq_current_is_worker(void)
{
	return in_task() && (current->flags & PF_IO_WORKER) &&
		current->worker_private;
}
#endif
Commit	Line	Data
771b53d0 JA	1	#ifndef INTERNAL_IO_WQ_H
	2	#define INTERNAL_IO_WQ_H
	3
e941894e	4	#include <linux/refcount.h>
98447d65	5
771b53d0 JA	6	struct io_wq;
	7
	8	enum {
	9	IO_WQ_WORK_CANCEL = 1,
e883a79d PB	10	IO_WQ_WORK_HASHED = 2,
e883a79d PB	11	IO_WQ_WORK_UNBOUND = 4,
e883a79d	12	IO_WQ_WORK_CONCURRENT = 16,
771b53d0 JA	13
	14	IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */
	15	};
	16
	17	enum io_wq_cancel {
	18	IO_WQ_CANCEL_OK, /* cancelled before started */
	19	IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */
	20	IO_WQ_CANCEL_NOTFOUND, /* work not found */
	21	};
	22
53e043b2 SM	23	struct io_wq_work_node {
	24	struct io_wq_work_node *next;
	25	};
	26
	27	struct io_wq_work_list {
	28	struct io_wq_work_node *first;
	29	struct io_wq_work_node *last;
	30	};
	31
0d9521b9 PB	32	#define wq_list_for_each(pos, prv, head) \
	33	for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
	34
5eef4e87 PB	35	#define wq_list_for_each_resume(pos, prv) \
	36	for (; pos; prv = pos, pos = (pos)->next)
	37
0d9521b9 PB	38	#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL)
	39	#define INIT_WQ_LIST(list) do { \
	40	(list)->first = NULL; \
0d9521b9 PB	41	} while (0)
0d9521b9 PB	42
86f3cd1b PB	43	static inline void wq_list_add_after(struct io_wq_work_node *node,
	44	struct io_wq_work_node *pos,
	45	struct io_wq_work_list *list)
	46	{
	47	struct io_wq_work_node *next = pos->next;
	48
	49	pos->next = node;
	50	node->next = next;
	51	if (!next)
	52	list->last = node;
	53	}
	54
24115c4e HX	55	/**
	56	* wq_list_merge - merge the second list to the first one.
	57	* @list0: the first list
	58	* @list1: the second list
	59	* Return the first node after mergence.
	60	*/
	61	static inline struct io_wq_work_node wq_list_merge(struct io_wq_work_list list0,
	62	struct io_wq_work_list *list1)
	63	{
	64	struct io_wq_work_node *ret;
	65
	66	if (!list0->first) {
	67	ret = list1->first;
	68	} else {
	69	ret = list0->first;
	70	list0->last->next = list1->first;
	71	}
	72	INIT_WQ_LIST(list0);
	73	INIT_WQ_LIST(list1);
	74	return ret;
	75	}
	76
6206f0e1 JA	77	static inline void wq_list_add_tail(struct io_wq_work_node *node,
	78	struct io_wq_work_list *list)
	79	{
8724dd8c	80	node->next = NULL;
6206f0e1	81	if (!list->first) {
e995d512 JA	82	list->last = node;
e995d512 JA	83	WRITE_ONCE(list->first, node);
6206f0e1 JA	84	} else {
	85	list->last->next = node;
	86	list->last = node;
	87	}
	88	}
	89
0d9521b9 PB	90	static inline void wq_list_add_head(struct io_wq_work_node *node,
	91	struct io_wq_work_list *list)
	92	{
	93	node->next = list->first;
	94	if (!node->next)
	95	list->last = node;
	96	WRITE_ONCE(list->first, node);
	97	}
	98
86f3cd1b PB	99	static inline void wq_list_cut(struct io_wq_work_list *list,
86f3cd1b PB	100	struct io_wq_work_node *last,
6206f0e1 JA	101	struct io_wq_work_node *prev)
6206f0e1 JA	102	{
86f3cd1b PB	103	/* first in the list, if prev==NULL */
	104	if (!prev)
	105	WRITE_ONCE(list->first, last->next);
	106	else
	107	prev->next = last->next;
	108
	109	if (last == list->last)
6206f0e1	110	list->last = prev;
86f3cd1b PB	111	last->next = NULL;
	112	}
	113
0d9521b9 PB	114	static inline void __wq_list_splice(struct io_wq_work_list *list,
	115	struct io_wq_work_node *to)
	116	{
	117	list->last->next = to->next;
	118	to->next = list->first;
	119	INIT_WQ_LIST(list);
	120	}
	121
	122	static inline bool wq_list_splice(struct io_wq_work_list *list,
	123	struct io_wq_work_node *to)
	124	{
	125	if (!wq_list_empty(list)) {
	126	__wq_list_splice(list, to);
	127	return true;
	128	}
	129	return false;
	130	}
	131
	132	static inline void wq_stack_add_head(struct io_wq_work_node *node,
	133	struct io_wq_work_node *stack)
	134	{
	135	node->next = stack->next;
	136	stack->next = node;
	137	}
	138
86f3cd1b PB	139	static inline void wq_list_del(struct io_wq_work_list *list,
	140	struct io_wq_work_node *node,
	141	struct io_wq_work_node *prev)
	142	{
	143	wq_list_cut(list, node, prev);
6206f0e1 JA	144	}
6206f0e1 JA	145
0d9521b9 PB	146	static inline
	147	struct io_wq_work_node wq_stack_extract(struct io_wq_work_node stack)
	148	{
	149	struct io_wq_work_node *node = stack->next;
6206f0e1	150
0d9521b9 PB	151	stack->next = node->next;
	152	return node;
	153	}
6206f0e1	154
771b53d0	155	struct io_wq_work {
18a542ff	156	struct io_wq_work_node list;
6206f0e1	157	unsigned flags;
771b53d0 JA	158	};
771b53d0 JA	159
86f3cd1b PB	160	static inline struct io_wq_work wq_next_work(struct io_wq_work work)
	161	{
	162	if (!work->list.next)
	163	return NULL;
	164
	165	return container_of(work->list.next, struct io_wq_work, list);
	166	}
	167
5280f7e5 PB	168	typedef struct io_wq_work (free_work_fn)(struct io_wq_work );
5280f7e5 PB	169	typedef void (io_wq_work_fn)(struct io_wq_work *);
7d723065	170
e941894e JA	171	struct io_wq_hash {
	172	refcount_t refs;
	173	unsigned long map;
	174	struct wait_queue_head wait;
	175	};
	176
	177	static inline void io_wq_put_hash(struct io_wq_hash *hash)
	178	{
	179	if (refcount_dec_and_test(&hash->refs))
	180	kfree(hash);
	181	}
	182
576a347b	183	struct io_wq_data {
e941894e	184	struct io_wq_hash *hash;
685fe7fe	185	struct task_struct *task;
f5fa38c5	186	io_wq_work_fn *do_work;
e9fd9396	187	free_work_fn *free_work;
576a347b JA	188	};
	189
	190	struct io_wq io_wq_create(unsigned bounded, struct io_wq_data data);
17a91051	191	void io_wq_exit_start(struct io_wq *wq);
afcc4015	192	void io_wq_put_and_exit(struct io_wq *wq);
771b53d0 JA	193
771b53d0 JA	194	void io_wq_enqueue(struct io_wq wq, struct io_wq_work work);
8766dd51 PB	195	void io_wq_hash_work(struct io_wq_work work, void val);
8766dd51 PB	196
fe76421d	197	int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask);
2e480058	198	int io_wq_max_workers(struct io_wq wq, int new_count);
fe76421d	199
8766dd51 PB	200	static inline bool io_wq_is_hashed(struct io_wq_work *work)
	201	{
	202	return work->flags & IO_WQ_WORK_HASHED;
	203	}
771b53d0	204
62755e35 JA	205	typedef bool (work_cancel_fn)(struct io_wq_work , void );
	206
	207	enum io_wq_cancel io_wq_cancel_cb(struct io_wq wq, work_cancel_fn cancel,
4f26bda1	208	void *data, bool cancel_all);
62755e35	209
771b53d0 JA	210	#if defined(CONFIG_IO_WQ)
	211	extern void io_wq_worker_sleeping(struct task_struct *);
	212	extern void io_wq_worker_running(struct task_struct *);
	213	#else
	214	static inline void io_wq_worker_sleeping(struct task_struct *tsk)
	215	{
	216	}
	217	static inline void io_wq_worker_running(struct task_struct *tsk)
	218	{
	219	}
525b305d	220	#endif
771b53d0	221
525b305d JA	222	static inline bool io_wq_current_is_worker(void)
525b305d JA	223	{
3bfe6106	224	return in_task() && (current->flags & PF_IO_WORKER) &&
e32cf5df	225	current->worker_private;
525b305d JA	226	}
525b305d JA	227	#endif