[thirdparty/kernel/stable.git] / block / blk-mq-sysfs.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"

static void blk_mq_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);

	free_percpu(ctxs->queue_ctx);
	kfree(ctxs);
}

static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);

	/* ctx->ctxs won't be released until all ctx are freed */
	kobject_put(&ctx->ctxs->kobj);
}

static void blk_mq_hw_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
						  kobj);

	cancel_delayed_work_sync(&hctx->run_work);

	if (hctx->flags & BLK_MQ_F_BLOCKING)
		cleanup_srcu_struct(hctx->srcu);
	blk_free_flush_queue(hctx->fq);
	sbitmap_free(&hctx->ctx_map);
	free_cpumask_var(hctx->cpumask);
	kfree(hctx->ctxs);
	kfree(hctx);
}

struct blk_mq_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_ctx *, char *);
	ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
};

struct blk_mq_hw_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
	ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
};

static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
				 char *page)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(ctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
				  const char *page, size_t length)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(ctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
				    struct attribute *attr, char *page)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(hctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
				     struct attribute *attr, const char *page,
				     size_t length)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(hctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
					    char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_tags);
}

static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
						     char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
}

static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	unsigned int i, first = 1;
	ssize_t ret = 0;

	for_each_cpu(i, hctx->cpumask) {
		if (first)
			ret += sprintf(ret + page, "%u", i);
		else
			ret += sprintf(ret + page, ", %u", i);

		first = 0;
	}

	ret += sprintf(ret + page, "\n");
	return ret;
}

static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	.attr = {.name = "nr_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	.attr = {.name = "cpu_list", .mode = 0444 },
	.show = blk_mq_hw_sysfs_cpus_show,
};

static struct attribute *default_hw_ctx_attrs[] = {
	&blk_mq_hw_sysfs_nr_tags.attr,
	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
	&blk_mq_hw_sysfs_cpus.attr,
	NULL,
};
ATTRIBUTE_GROUPS(default_hw_ctx);

static const struct sysfs_ops blk_mq_sysfs_ops = {
	.show	= blk_mq_sysfs_show,
	.store	= blk_mq_sysfs_store,
};

static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	.show	= blk_mq_hw_sysfs_show,
	.store	= blk_mq_hw_sysfs_store,
};

static struct kobj_type blk_mq_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_ctx_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_ctx_sysfs_release,
};

static struct kobj_type blk_mq_hw_ktype = {
	.sysfs_ops	= &blk_mq_hw_sysfs_ops,
	.default_groups = default_hw_ctx_groups,
	.release	= blk_mq_hw_sysfs_release,
};

static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	int i;

	if (!hctx->nr_ctx)
		return;

	hctx_for_each_ctx(hctx, ctx, i)
		kobject_del(&ctx->kobj);

	kobject_del(&hctx->kobj);
}

static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	int i, ret;

	if (!hctx->nr_ctx)
		return 0;

	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
	if (ret)
		return ret;

	hctx_for_each_ctx(hctx, ctx, i) {
		ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
		if (ret)
			break;
	}

	return ret;
}

void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	lockdep_assert_held(&q->sysfs_lock);

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);
	kobject_put(&dev->kobj);

	q->mq_sysfs_init_done = false;
}

void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}

void blk_mq_sysfs_deinit(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_put(&ctx->kobj);
	}
	kobject_put(q->mq_kobj);
}

void blk_mq_sysfs_init(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	kobject_init(q->mq_kobj, &blk_mq_ktype);

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);

		kobject_get(q->mq_kobj);
		kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
	}
}

int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int ret, i;

	WARN_ON_ONCE(!q->kobj.parent);
	lockdep_assert_held(&q->sysfs_lock);

	ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	if (ret < 0)
		goto out;

	kobject_uevent(q->mq_kobj, KOBJ_ADD);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			goto unreg;
	}

	q->mq_sysfs_init_done = true;

out:
	return ret;

unreg:
	while (--i >= 0)
		blk_mq_unregister_hctx(q->queue_hw_ctx[i]);

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);
	kobject_put(&dev->kobj);
	return ret;
}

int blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	int ret;

	mutex_lock(&q->sysfs_lock);
	ret = __blk_mq_register_dev(dev, q);
	mutex_unlock(&q->sysfs_lock);

	return ret;
}

void blk_mq_sysfs_unregister(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

unlock:
	mutex_unlock(&q->sysfs_lock);
}

int blk_mq_sysfs_register(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i, ret = 0;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

unlock:
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
Commit	Line	Data
3dcf60bc	1	// SPDX-License-Identifier: GPL-2.0
320ae51f JA	2	#include <linux/kernel.h>
	3	#include <linux/module.h>
	4	#include <linux/backing-dev.h>
	5	#include <linux/bio.h>
	6	#include <linux/blkdev.h>
	7	#include <linux/mm.h>
	8	#include <linux/init.h>
	9	#include <linux/slab.h>
	10	#include <linux/workqueue.h>
	11	#include <linux/smp.h>
	12
	13	#include <linux/blk-mq.h>
c7e2d94b	14	#include "blk.h"
320ae51f JA	15	#include "blk-mq.h"
	16	#include "blk-mq-tag.h"
	17
	18	static void blk_mq_sysfs_release(struct kobject *kobj)
	19	{
1db4909e ML	20	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);
	21
	22	free_percpu(ctxs->queue_ctx);
	23	kfree(ctxs);
	24	}
	25
	26	static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
	27	{
	28	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	29
	30	/* ctx->ctxs won't be released until all ctx are freed */
	31	kobject_put(&ctx->ctxs->kobj);
320ae51f JA	32	}
320ae51f JA	33
6c8b232e ML	34	static void blk_mq_hw_sysfs_release(struct kobject *kobj)
	35	{
	36	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
	37	kobj);
c7e2d94b	38
1b97871b ML	39	cancel_delayed_work_sync(&hctx->run_work);
1b97871b ML	40
c7e2d94b ML	41	if (hctx->flags & BLK_MQ_F_BLOCKING)
	42	cleanup_srcu_struct(hctx->srcu);
	43	blk_free_flush_queue(hctx->fq);
	44	sbitmap_free(&hctx->ctx_map);
01388df3	45	free_cpumask_var(hctx->cpumask);
6c8b232e ML	46	kfree(hctx->ctxs);
	47	kfree(hctx);
	48	}
	49
320ae51f JA	50	struct blk_mq_ctx_sysfs_entry {
	51	struct attribute attr;
	52	ssize_t (show)(struct blk_mq_ctx , char *);
	53	ssize_t (store)(struct blk_mq_ctx , const char *, size_t);
	54	};
	55
	56	struct blk_mq_hw_ctx_sysfs_entry {
	57	struct attribute attr;
	58	ssize_t (show)(struct blk_mq_hw_ctx , char *);
	59	ssize_t (store)(struct blk_mq_hw_ctx , const char *, size_t);
	60	};
	61
	62	static ssize_t blk_mq_sysfs_show(struct kobject kobj, struct attribute attr,
	63	char *page)
	64	{
	65	struct blk_mq_ctx_sysfs_entry *entry;
	66	struct blk_mq_ctx *ctx;
	67	struct request_queue *q;
	68	ssize_t res;
	69
	70	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	71	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	72	q = ctx->queue;
	73
	74	if (!entry->show)
	75	return -EIO;
	76
	77	res = -ENOENT;
	78	mutex_lock(&q->sysfs_lock);
	79	if (!blk_queue_dying(q))
	80	res = entry->show(ctx, page);
	81	mutex_unlock(&q->sysfs_lock);
	82	return res;
	83	}
	84
	85	static ssize_t blk_mq_sysfs_store(struct kobject kobj, struct attribute attr,
	86	const char *page, size_t length)
	87	{
	88	struct blk_mq_ctx_sysfs_entry *entry;
	89	struct blk_mq_ctx *ctx;
	90	struct request_queue *q;
	91	ssize_t res;
	92
	93	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	94	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	95	q = ctx->queue;
	96
	97	if (!entry->store)
	98	return -EIO;
	99
	100	res = -ENOENT;
	101	mutex_lock(&q->sysfs_lock);
	102	if (!blk_queue_dying(q))
	103	res = entry->store(ctx, page, length);
	104	mutex_unlock(&q->sysfs_lock);
	105	return res;
	106	}
	107
	108	static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
	109	struct attribute attr, char page)
	110	{
	111	struct blk_mq_hw_ctx_sysfs_entry *entry;
	112	struct blk_mq_hw_ctx *hctx;
	113	struct request_queue *q;
114	ssize_t res;
115
116	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
117	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
118	q = hctx->queue;
119
120	if (!entry->show)
121	return -EIO;
122
123	res = -ENOENT;
124	mutex_lock(&q->sysfs_lock);
125	if (!blk_queue_dying(q))
126	res = entry->show(hctx, page);
127	mutex_unlock(&q->sysfs_lock);
128	return res;
129	}
130
131	static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
132	struct attribute attr, const char page,
133	size_t length)
134	{
135	struct blk_mq_hw_ctx_sysfs_entry *entry;
136	struct blk_mq_hw_ctx *hctx;
137	struct request_queue *q;
138	ssize_t res;
139
140	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
141	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
142	q = hctx->queue;
143
144	if (!entry->store)
145	return -EIO;
146
147	res = -ENOENT;
148	mutex_lock(&q->sysfs_lock);
149	if (!blk_queue_dying(q))
150	res = entry->store(hctx, page, length);
151	mutex_unlock(&q->sysfs_lock);
152	return res;
153	}
154
d96b37c0 OS	155	static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	156	char *page)
bd166ef1	157	{
d96b37c0	158	return sprintf(page, "%u\n", hctx->tags->nr_tags);
bd166ef1 JA	159	}
bd166ef1 JA	160
d96b37c0 OS	161	static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	162	char *page)
320ae51f	163	{
d96b37c0	164	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
320ae51f JA	165	}
320ae51f JA	166
676141e4 JA	167	static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx hctx, char page)
676141e4 JA	168	{
cb2da43e	169	unsigned int i, first = 1;
676141e4 JA	170	ssize_t ret = 0;
676141e4 JA	171
cb2da43e	172	for_each_cpu(i, hctx->cpumask) {
676141e4 JA	173	if (first)
	174	ret += sprintf(ret + page, "%u", i);
	175	else
	176	ret += sprintf(ret + page, ", %u", i);
	177
	178	first = 0;
	179	}
	180
676141e4 JA	181	ret += sprintf(ret + page, "\n");
	182	return ret;
	183	}
	184
d96b37c0	185	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
5657a819	186	.attr = {.name = "nr_tags", .mode = 0444 },
d96b37c0 OS	187	.show = blk_mq_hw_sysfs_nr_tags_show,
	188	};
	189	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
5657a819	190	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
d96b37c0 OS	191	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
d96b37c0 OS	192	};
676141e4	193	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
5657a819	194	.attr = {.name = "cpu_list", .mode = 0444 },
676141e4 JA	195	.show = blk_mq_hw_sysfs_cpus_show,
676141e4 JA	196	};
320ae51f JA	197
320ae51f JA	198	static struct attribute *default_hw_ctx_attrs[] = {
d96b37c0 OS	199	&blk_mq_hw_sysfs_nr_tags.attr,
d96b37c0 OS	200	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
676141e4	201	&blk_mq_hw_sysfs_cpus.attr,
320ae51f JA	202	NULL,
320ae51f JA	203	};
800f5aa1	204	ATTRIBUTE_GROUPS(default_hw_ctx);
320ae51f JA	205
	206	static const struct sysfs_ops blk_mq_sysfs_ops = {
	207	.show = blk_mq_sysfs_show,
	208	.store = blk_mq_sysfs_store,
	209	};
	210
	211	static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	212	.show = blk_mq_hw_sysfs_show,
	213	.store = blk_mq_hw_sysfs_store,
	214	};
	215
	216	static struct kobj_type blk_mq_ktype = {
	217	.sysfs_ops = &blk_mq_sysfs_ops,
	218	.release = blk_mq_sysfs_release,
	219	};
	220
	221	static struct kobj_type blk_mq_ctx_ktype = {
	222	.sysfs_ops = &blk_mq_sysfs_ops,
1db4909e	223	.release = blk_mq_ctx_sysfs_release,
320ae51f JA	224	};
	225
	226	static struct kobj_type blk_mq_hw_ktype = {
	227	.sysfs_ops = &blk_mq_hw_sysfs_ops,
800f5aa1	228	.default_groups = default_hw_ctx_groups,
6c8b232e	229	.release = blk_mq_hw_sysfs_release,
320ae51f JA	230	};
320ae51f JA	231
ee3c5db0	232	static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	233	{
	234	struct blk_mq_ctx *ctx;
	235	int i;
	236
4593fdbe	237	if (!hctx->nr_ctx)
67aec14c JA	238	return;
	239
	240	hctx_for_each_ctx(hctx, ctx, i)
	241	kobject_del(&ctx->kobj);
	242
	243	kobject_del(&hctx->kobj);
	244	}
	245
ee3c5db0	246	static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	247	{
	248	struct request_queue *q = hctx->queue;
	249	struct blk_mq_ctx *ctx;
	250	int i, ret;
	251
4593fdbe	252	if (!hctx->nr_ctx)
67aec14c JA	253	return 0;
67aec14c JA	254
1db4909e	255	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
67aec14c JA	256	if (ret)
	257	return ret;
	258
	259	hctx_for_each_ctx(hctx, ctx, i) {
	260	ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
	261	if (ret)
	262	break;
	263	}
	264
	265	return ret;
	266	}
	267
667257e8	268	void blk_mq_unregister_dev(struct device dev, struct request_queue q)
320ae51f	269	{
85157366	270	struct blk_mq_hw_ctx *hctx;
7ea5fe31	271	int i;
85157366	272
2d0364c8 BVA	273	lockdep_assert_held(&q->sysfs_lock);
2d0364c8 BVA	274
6c8b232e	275	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	276	blk_mq_unregister_hctx(hctx);
67aec14c JA	277
1db4909e ML	278	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
1db4909e ML	279	kobject_del(q->mq_kobj);
b21d5b30	280	kobject_put(&dev->kobj);
4593fdbe AM	281
4593fdbe AM	282	q->mq_sysfs_init_done = false;
c0f3fd2b JA	283	}
c0f3fd2b JA	284
868f2f0b KB	285	void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
	286	{
	287	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
	288	}
	289
7ea5fe31 ML	290	void blk_mq_sysfs_deinit(struct request_queue *q)
	291	{
	292	struct blk_mq_ctx *ctx;
	293	int cpu;
	294
	295	for_each_possible_cpu(cpu) {
	296	ctx = per_cpu_ptr(q->queue_ctx, cpu);
	297	kobject_put(&ctx->kobj);
	298	}
1db4909e	299	kobject_put(q->mq_kobj);
7ea5fe31 ML	300	}
7ea5fe31 ML	301
737f98cf	302	void blk_mq_sysfs_init(struct request_queue *q)
67aec14c	303	{
67aec14c	304	struct blk_mq_ctx *ctx;
897bb0c7	305	int cpu;
67aec14c	306
1db4909e	307	kobject_init(q->mq_kobj, &blk_mq_ktype);
67aec14c	308
897bb0c7 TG	309	for_each_possible_cpu(cpu) {
897bb0c7 TG	310	ctx = per_cpu_ptr(q->queue_ctx, cpu);
1db4909e ML	311
1db4909e ML	312	kobject_get(q->mq_kobj);
06a41a99	313	kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
897bb0c7	314	}
67aec14c JA	315	}
67aec14c JA	316
2d0364c8	317	int __blk_mq_register_dev(struct device dev, struct request_queue q)
320ae51f	318	{
320ae51f	319	struct blk_mq_hw_ctx *hctx;
67aec14c	320	int ret, i;
320ae51f	321
2d0364c8 BVA	322	WARN_ON_ONCE(!q->kobj.parent);
2d0364c8 BVA	323	lockdep_assert_held(&q->sysfs_lock);
4593fdbe	324
1db4909e	325	ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
320ae51f	326	if (ret < 0)
4593fdbe	327	goto out;
320ae51f	328
1db4909e	329	kobject_uevent(q->mq_kobj, KOBJ_ADD);
320ae51f JA	330
320ae51f JA	331	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c	332	ret = blk_mq_register_hctx(hctx);
320ae51f	333	if (ret)
f05d1ba7	334	goto unreg;
320ae51f JA	335	}
320ae51f JA	336
f05d1ba7	337	q->mq_sysfs_init_done = true;
2d0364c8	338
4593fdbe	339	out:
2d0364c8	340	return ret;
f05d1ba7 BVA	341
	342	unreg:
	343	while (--i >= 0)
	344	blk_mq_unregister_hctx(q->queue_hw_ctx[i]);
	345
1db4909e ML	346	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
1db4909e ML	347	kobject_del(q->mq_kobj);
f05d1ba7 BVA	348	kobject_put(&dev->kobj);
f05d1ba7 BVA	349	return ret;
2d0364c8 BVA	350	}
	351
	352	int blk_mq_register_dev(struct device dev, struct request_queue q)
	353	{
	354	int ret;
	355
	356	mutex_lock(&q->sysfs_lock);
	357	ret = __blk_mq_register_dev(dev, q);
	358	mutex_unlock(&q->sysfs_lock);
320ae51f	359
4593fdbe	360	return ret;
320ae51f	361	}
67aec14c JA	362
	363	void blk_mq_sysfs_unregister(struct request_queue *q)
	364	{
	365	struct blk_mq_hw_ctx *hctx;
	366	int i;
	367
2d0364c8	368	mutex_lock(&q->sysfs_lock);
4593fdbe	369	if (!q->mq_sysfs_init_done)
2d0364c8	370	goto unlock;
4593fdbe	371
67aec14c JA	372	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	373	blk_mq_unregister_hctx(hctx);
2d0364c8 BVA	374
	375	unlock:
	376	mutex_unlock(&q->sysfs_lock);
67aec14c JA	377	}
	378
	379	int blk_mq_sysfs_register(struct request_queue *q)
	380	{
	381	struct blk_mq_hw_ctx *hctx;
	382	int i, ret = 0;
	383
2d0364c8	384	mutex_lock(&q->sysfs_lock);
4593fdbe	385	if (!q->mq_sysfs_init_done)
2d0364c8	386	goto unlock;
4593fdbe	387
67aec14c JA	388	queue_for_each_hw_ctx(q, hctx, i) {
	389	ret = blk_mq_register_hctx(hctx);
	390	if (ret)
	391	break;
	392	}
	393
2d0364c8 BVA	394	unlock:
	395	mutex_unlock(&q->sysfs_lock);
	396
67aec14c JA	397	return ret;
67aec14c JA	398	}