[thirdparty/kernel/stable.git] / block / blk-mq-sysfs.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>

#include <linux/blk-mq.h>
#include "blk-mq.h"
#include "blk-mq-tag.h"

static void blk_mq_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);

	free_percpu(ctxs->queue_ctx);
	kfree(ctxs);
}

static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);

	/* ctx->ctxs won't be released until all ctx are freed */
	kobject_put(&ctx->ctxs->kobj);
}

static void blk_mq_hw_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
						  kobj);
	free_cpumask_var(hctx->cpumask);
	kfree(hctx->ctxs);
	kfree(hctx);
}

struct blk_mq_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_ctx *, char *);
	ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
};

struct blk_mq_hw_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
	ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
};

static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
				 char *page)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(ctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
				  const char *page, size_t length)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(ctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
				    struct attribute *attr, char *page)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(hctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
				     struct attribute *attr, const char *page,
				     size_t length)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(hctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
					    char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_tags);
}

static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
						     char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
}

static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	unsigned int i, first = 1;
	ssize_t ret = 0;

	for_each_cpu(i, hctx->cpumask) {
		if (first)
			ret += sprintf(ret + page, "%u", i);
		else
			ret += sprintf(ret + page, ", %u", i);

		first = 0;
	}

	ret += sprintf(ret + page, "\n");
	return ret;
}

static struct attribute *default_ctx_attrs[] = {
	NULL,
};

static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	.attr = {.name = "nr_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	.attr = {.name = "cpu_list", .mode = 0444 },
	.show = blk_mq_hw_sysfs_cpus_show,
};

static struct attribute *default_hw_ctx_attrs[] = {
	&blk_mq_hw_sysfs_nr_tags.attr,
	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
	&blk_mq_hw_sysfs_cpus.attr,
	NULL,
};

static const struct sysfs_ops blk_mq_sysfs_ops = {
	.show	= blk_mq_sysfs_show,
	.store	= blk_mq_sysfs_store,
};

static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	.show	= blk_mq_hw_sysfs_show,
	.store	= blk_mq_hw_sysfs_store,
};

static struct kobj_type blk_mq_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_ctx_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.default_attrs	= default_ctx_attrs,
	.release	= blk_mq_ctx_sysfs_release,
};

static struct kobj_type blk_mq_hw_ktype = {
	.sysfs_ops	= &blk_mq_hw_sysfs_ops,
	.default_attrs	= default_hw_ctx_attrs,
	.release	= blk_mq_hw_sysfs_release,
};

static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	int i;

	if (!hctx->nr_ctx)
		return;

	hctx_for_each_ctx(hctx, ctx, i)
		kobject_del(&ctx->kobj);

	kobject_del(&hctx->kobj);
}

static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	int i, ret;

	if (!hctx->nr_ctx)
		return 0;

	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
	if (ret)
		return ret;

	hctx_for_each_ctx(hctx, ctx, i) {
		ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
		if (ret)
			break;
	}

	return ret;
}

void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	lockdep_assert_held(&q->sysfs_lock);

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);
	kobject_put(&dev->kobj);

	q->mq_sysfs_init_done = false;
}

void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}

void blk_mq_sysfs_deinit(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_put(&ctx->kobj);
	}
	kobject_put(q->mq_kobj);
}

void blk_mq_sysfs_init(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	kobject_init(q->mq_kobj, &blk_mq_ktype);

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);

		kobject_get(q->mq_kobj);
		kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
	}
}

int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int ret, i;

	WARN_ON_ONCE(!q->kobj.parent);
	lockdep_assert_held(&q->sysfs_lock);

	ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	if (ret < 0)
		goto out;

	kobject_uevent(q->mq_kobj, KOBJ_ADD);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			goto unreg;
	}

	q->mq_sysfs_init_done = true;

out:
	return ret;

unreg:
	while (--i >= 0)
		blk_mq_unregister_hctx(q->queue_hw_ctx[i]);

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);
	kobject_put(&dev->kobj);
	return ret;
}

int blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	int ret;

	mutex_lock(&q->sysfs_lock);
	ret = __blk_mq_register_dev(dev, q);
	mutex_unlock(&q->sysfs_lock);

	return ret;
}

void blk_mq_sysfs_unregister(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

unlock:
	mutex_unlock(&q->sysfs_lock);
}

int blk_mq_sysfs_register(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i, ret = 0;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

unlock:
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
Commit	Line	Data
3dcf60bc	1	// SPDX-License-Identifier: GPL-2.0
320ae51f JA	2	#include <linux/kernel.h>
	3	#include <linux/module.h>
	4	#include <linux/backing-dev.h>
	5	#include <linux/bio.h>
	6	#include <linux/blkdev.h>
	7	#include <linux/mm.h>
	8	#include <linux/init.h>
	9	#include <linux/slab.h>
	10	#include <linux/workqueue.h>
	11	#include <linux/smp.h>
	12
	13	#include <linux/blk-mq.h>
	14	#include "blk-mq.h"
	15	#include "blk-mq-tag.h"
	16
	17	static void blk_mq_sysfs_release(struct kobject *kobj)
	18	{
1db4909e ML	19	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);
	20
	21	free_percpu(ctxs->queue_ctx);
	22	kfree(ctxs);
	23	}
	24
	25	static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
	26	{
	27	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	28
	29	/* ctx->ctxs won't be released until all ctx are freed */
	30	kobject_put(&ctx->ctxs->kobj);
320ae51f JA	31	}
320ae51f JA	32
6c8b232e ML	33	static void blk_mq_hw_sysfs_release(struct kobject *kobj)
	34	{
	35	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
	36	kobj);
01388df3	37	free_cpumask_var(hctx->cpumask);
6c8b232e ML	38	kfree(hctx->ctxs);
	39	kfree(hctx);
	40	}
	41
320ae51f JA	42	struct blk_mq_ctx_sysfs_entry {
	43	struct attribute attr;
	44	ssize_t (show)(struct blk_mq_ctx , char *);
	45	ssize_t (store)(struct blk_mq_ctx , const char *, size_t);
	46	};
	47
	48	struct blk_mq_hw_ctx_sysfs_entry {
	49	struct attribute attr;
	50	ssize_t (show)(struct blk_mq_hw_ctx , char *);
	51	ssize_t (store)(struct blk_mq_hw_ctx , const char *, size_t);
	52	};
	53
	54	static ssize_t blk_mq_sysfs_show(struct kobject kobj, struct attribute attr,
	55	char *page)
	56	{
	57	struct blk_mq_ctx_sysfs_entry *entry;
	58	struct blk_mq_ctx *ctx;
	59	struct request_queue *q;
	60	ssize_t res;
	61
	62	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	63	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	64	q = ctx->queue;
	65
	66	if (!entry->show)
	67	return -EIO;
	68
	69	res = -ENOENT;
	70	mutex_lock(&q->sysfs_lock);
	71	if (!blk_queue_dying(q))
	72	res = entry->show(ctx, page);
	73	mutex_unlock(&q->sysfs_lock);
	74	return res;
	75	}
	76
	77	static ssize_t blk_mq_sysfs_store(struct kobject kobj, struct attribute attr,
	78	const char *page, size_t length)
	79	{
	80	struct blk_mq_ctx_sysfs_entry *entry;
	81	struct blk_mq_ctx *ctx;
	82	struct request_queue *q;
	83	ssize_t res;
	84
	85	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	86	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	87	q = ctx->queue;
	88
	89	if (!entry->store)
	90	return -EIO;
	91
	92	res = -ENOENT;
	93	mutex_lock(&q->sysfs_lock);
	94	if (!blk_queue_dying(q))
	95	res = entry->store(ctx, page, length);
	96	mutex_unlock(&q->sysfs_lock);
	97	return res;
	98	}
	99
	100	static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
	101	struct attribute attr, char page)
	102	{
	103	struct blk_mq_hw_ctx_sysfs_entry *entry;
	104	struct blk_mq_hw_ctx *hctx;
	105	struct request_queue *q;
106	ssize_t res;
107
108	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
109	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
110	q = hctx->queue;
111
112	if (!entry->show)
113	return -EIO;
114
115	res = -ENOENT;
116	mutex_lock(&q->sysfs_lock);
117	if (!blk_queue_dying(q))
118	res = entry->show(hctx, page);
119	mutex_unlock(&q->sysfs_lock);
120	return res;
121	}
122
123	static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
124	struct attribute attr, const char page,
125	size_t length)
126	{
127	struct blk_mq_hw_ctx_sysfs_entry *entry;
128	struct blk_mq_hw_ctx *hctx;
129	struct request_queue *q;
130	ssize_t res;
131
132	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
133	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
134	q = hctx->queue;
135
136	if (!entry->store)
137	return -EIO;
138
139	res = -ENOENT;
140	mutex_lock(&q->sysfs_lock);
141	if (!blk_queue_dying(q))
142	res = entry->store(hctx, page, length);
143	mutex_unlock(&q->sysfs_lock);
144	return res;
145	}
146
d96b37c0 OS	147	static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	148	char *page)
bd166ef1	149	{
d96b37c0	150	return sprintf(page, "%u\n", hctx->tags->nr_tags);
bd166ef1 JA	151	}
bd166ef1 JA	152
d96b37c0 OS	153	static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	154	char *page)
320ae51f	155	{
d96b37c0	156	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
320ae51f JA	157	}
320ae51f JA	158
676141e4 JA	159	static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx hctx, char page)
676141e4 JA	160	{
cb2da43e	161	unsigned int i, first = 1;
676141e4 JA	162	ssize_t ret = 0;
676141e4 JA	163
cb2da43e	164	for_each_cpu(i, hctx->cpumask) {
676141e4 JA	165	if (first)
	166	ret += sprintf(ret + page, "%u", i);
	167	else
	168	ret += sprintf(ret + page, ", %u", i);
	169
	170	first = 0;
	171	}
	172
676141e4 JA	173	ret += sprintf(ret + page, "\n");
	174	return ret;
	175	}
	176
320ae51f	177	static struct attribute *default_ctx_attrs[] = {
320ae51f JA	178	NULL,
	179	};
	180
d96b37c0	181	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
5657a819	182	.attr = {.name = "nr_tags", .mode = 0444 },
d96b37c0 OS	183	.show = blk_mq_hw_sysfs_nr_tags_show,
	184	};
	185	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
5657a819	186	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
d96b37c0 OS	187	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
d96b37c0 OS	188	};
676141e4	189	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
5657a819	190	.attr = {.name = "cpu_list", .mode = 0444 },
676141e4 JA	191	.show = blk_mq_hw_sysfs_cpus_show,
676141e4 JA	192	};
320ae51f JA	193
320ae51f JA	194	static struct attribute *default_hw_ctx_attrs[] = {
d96b37c0 OS	195	&blk_mq_hw_sysfs_nr_tags.attr,
d96b37c0 OS	196	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
676141e4	197	&blk_mq_hw_sysfs_cpus.attr,
320ae51f JA	198	NULL,
	199	};
	200
	201	static const struct sysfs_ops blk_mq_sysfs_ops = {
	202	.show = blk_mq_sysfs_show,
	203	.store = blk_mq_sysfs_store,
	204	};
	205
	206	static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	207	.show = blk_mq_hw_sysfs_show,
	208	.store = blk_mq_hw_sysfs_store,
	209	};
	210
	211	static struct kobj_type blk_mq_ktype = {
	212	.sysfs_ops = &blk_mq_sysfs_ops,
	213	.release = blk_mq_sysfs_release,
	214	};
	215
	216	static struct kobj_type blk_mq_ctx_ktype = {
	217	.sysfs_ops = &blk_mq_sysfs_ops,
	218	.default_attrs = default_ctx_attrs,
1db4909e	219	.release = blk_mq_ctx_sysfs_release,
320ae51f JA	220	};
	221
	222	static struct kobj_type blk_mq_hw_ktype = {
	223	.sysfs_ops = &blk_mq_hw_sysfs_ops,
	224	.default_attrs = default_hw_ctx_attrs,
6c8b232e	225	.release = blk_mq_hw_sysfs_release,
320ae51f JA	226	};
320ae51f JA	227
ee3c5db0	228	static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	229	{
	230	struct blk_mq_ctx *ctx;
	231	int i;
	232
4593fdbe	233	if (!hctx->nr_ctx)
67aec14c JA	234	return;
	235
	236	hctx_for_each_ctx(hctx, ctx, i)
	237	kobject_del(&ctx->kobj);
	238
	239	kobject_del(&hctx->kobj);
	240	}
	241
ee3c5db0	242	static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	243	{
	244	struct request_queue *q = hctx->queue;
	245	struct blk_mq_ctx *ctx;
	246	int i, ret;
	247
4593fdbe	248	if (!hctx->nr_ctx)
67aec14c JA	249	return 0;
67aec14c JA	250
1db4909e	251	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
67aec14c JA	252	if (ret)
	253	return ret;
	254
	255	hctx_for_each_ctx(hctx, ctx, i) {
	256	ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
	257	if (ret)
	258	break;
	259	}
	260
	261	return ret;
	262	}
	263
667257e8	264	void blk_mq_unregister_dev(struct device dev, struct request_queue q)
320ae51f	265	{
85157366	266	struct blk_mq_hw_ctx *hctx;
7ea5fe31	267	int i;
85157366	268
2d0364c8 BVA	269	lockdep_assert_held(&q->sysfs_lock);
2d0364c8 BVA	270
6c8b232e	271	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	272	blk_mq_unregister_hctx(hctx);
67aec14c JA	273
1db4909e ML	274	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
1db4909e ML	275	kobject_del(q->mq_kobj);
b21d5b30	276	kobject_put(&dev->kobj);
4593fdbe AM	277
4593fdbe AM	278	q->mq_sysfs_init_done = false;
c0f3fd2b JA	279	}
c0f3fd2b JA	280
868f2f0b KB	281	void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
	282	{
	283	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
	284	}
	285
7ea5fe31 ML	286	void blk_mq_sysfs_deinit(struct request_queue *q)
	287	{
	288	struct blk_mq_ctx *ctx;
	289	int cpu;
	290
	291	for_each_possible_cpu(cpu) {
	292	ctx = per_cpu_ptr(q->queue_ctx, cpu);
	293	kobject_put(&ctx->kobj);
	294	}
1db4909e	295	kobject_put(q->mq_kobj);
7ea5fe31 ML	296	}
7ea5fe31 ML	297
737f98cf	298	void blk_mq_sysfs_init(struct request_queue *q)
67aec14c	299	{
67aec14c	300	struct blk_mq_ctx *ctx;
897bb0c7	301	int cpu;
67aec14c	302
1db4909e	303	kobject_init(q->mq_kobj, &blk_mq_ktype);
67aec14c	304
897bb0c7 TG	305	for_each_possible_cpu(cpu) {
897bb0c7 TG	306	ctx = per_cpu_ptr(q->queue_ctx, cpu);
1db4909e ML	307
1db4909e ML	308	kobject_get(q->mq_kobj);
06a41a99	309	kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
897bb0c7	310	}
67aec14c JA	311	}
67aec14c JA	312
2d0364c8	313	int __blk_mq_register_dev(struct device dev, struct request_queue q)
320ae51f	314	{
320ae51f	315	struct blk_mq_hw_ctx *hctx;
67aec14c	316	int ret, i;
320ae51f	317
2d0364c8 BVA	318	WARN_ON_ONCE(!q->kobj.parent);
2d0364c8 BVA	319	lockdep_assert_held(&q->sysfs_lock);
4593fdbe	320
1db4909e	321	ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
320ae51f	322	if (ret < 0)
4593fdbe	323	goto out;
320ae51f	324
1db4909e	325	kobject_uevent(q->mq_kobj, KOBJ_ADD);
320ae51f JA	326
320ae51f JA	327	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c	328	ret = blk_mq_register_hctx(hctx);
320ae51f	329	if (ret)
f05d1ba7	330	goto unreg;
320ae51f JA	331	}
320ae51f JA	332
f05d1ba7	333	q->mq_sysfs_init_done = true;
2d0364c8	334
4593fdbe	335	out:
2d0364c8	336	return ret;
f05d1ba7 BVA	337
	338	unreg:
	339	while (--i >= 0)
	340	blk_mq_unregister_hctx(q->queue_hw_ctx[i]);
	341
1db4909e ML	342	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
1db4909e ML	343	kobject_del(q->mq_kobj);
f05d1ba7 BVA	344	kobject_put(&dev->kobj);
f05d1ba7 BVA	345	return ret;
2d0364c8 BVA	346	}
	347
	348	int blk_mq_register_dev(struct device dev, struct request_queue q)
	349	{
	350	int ret;
	351
	352	mutex_lock(&q->sysfs_lock);
	353	ret = __blk_mq_register_dev(dev, q);
	354	mutex_unlock(&q->sysfs_lock);
320ae51f	355
4593fdbe	356	return ret;
320ae51f	357	}
67aec14c JA	358
	359	void blk_mq_sysfs_unregister(struct request_queue *q)
	360	{
	361	struct blk_mq_hw_ctx *hctx;
	362	int i;
	363
2d0364c8	364	mutex_lock(&q->sysfs_lock);
4593fdbe	365	if (!q->mq_sysfs_init_done)
2d0364c8	366	goto unlock;
4593fdbe	367
67aec14c JA	368	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	369	blk_mq_unregister_hctx(hctx);
2d0364c8 BVA	370
	371	unlock:
	372	mutex_unlock(&q->sysfs_lock);
67aec14c JA	373	}
	374
	375	int blk_mq_sysfs_register(struct request_queue *q)
	376	{
	377	struct blk_mq_hw_ctx *hctx;
	378	int i, ret = 0;
	379
2d0364c8	380	mutex_lock(&q->sysfs_lock);
4593fdbe	381	if (!q->mq_sysfs_init_done)
2d0364c8	382	goto unlock;
4593fdbe	383
67aec14c JA	384	queue_for_each_hw_ctx(q, hctx, i) {
	385	ret = blk_mq_register_hctx(hctx);
	386	if (ret)
	387	break;
	388	}
	389
2d0364c8 BVA	390	unlock:
	391	mutex_unlock(&q->sysfs_lock);
	392
67aec14c JA	393	return ret;
67aec14c JA	394	}