block: use {alloc|free}_sched data methods

The previous patch introduced ->alloc_sched_data and
->free_sched_data methods. This patch builds upon that
by now using these methods during elevator switch and
nr_hw_queue update.

It's also ensured that scheduler-specific data is
allocated and freed through the new callbacks outside
of the ->freeze_lock and ->elevator_lock locking contexts,
thereby preventing any dependency on pcpu_alloc_mutex.

Reviewed-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Yu Kuai <yukuai@fnnas.com>
Signed-off-by: Nilay Shroff <nilay@linux.ibm.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 32ae57518298d455155278cb26e9509c97afeb0e..e26898128a7e53076830302e13bdab67db6a7ba5 100644 (file)
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -428,12 +428,17 @@ void blk_mq_free_sched_tags(struct elevator_tags *et,
 }
 
 void blk_mq_free_sched_res(struct elevator_resources *res,
+               struct elevator_type *type,
                struct blk_mq_tag_set *set)
 {
        if (res->et) {
                blk_mq_free_sched_tags(res->et, set);
                res->et = NULL;
        }
+       if (res->data) {
+               blk_mq_free_sched_data(type, res->data);
+               res->data = NULL;
+       }
 }
 
 void blk_mq_free_sched_res_batch(struct xarray *elv_tbl,
@@ -458,7 +463,7 @@ void blk_mq_free_sched_res_batch(struct xarray *elv_tbl,
                                WARN_ON_ONCE(1);
                                continue;
                        }
-                       blk_mq_free_sched_res(&ctx->res, set);
+                       blk_mq_free_sched_res(&ctx->res, ctx->type, set);
                }
        }
 }
@@ -540,7 +545,9 @@ out:
 }
 
 int blk_mq_alloc_sched_res(struct request_queue *q,
-               struct elevator_resources *res, unsigned int nr_hw_queues)
+               struct elevator_type *type,
+               struct elevator_resources *res,
+               unsigned int nr_hw_queues)
 {
        struct blk_mq_tag_set *set = q->tag_set;
 
@@ -549,6 +556,12 @@ int blk_mq_alloc_sched_res(struct request_queue *q,
        if (!res->et)
                return -ENOMEM;
 
+       res->data = blk_mq_alloc_sched_data(q, type);
+       if (IS_ERR(res->data)) {
+               blk_mq_free_sched_tags(res->et, set);
+               return -ENOMEM;
+       }
+
        return 0;
 }
 
@@ -576,19 +589,21 @@ int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl,
                                goto out_unwind;
                        }
 
-                       ret = blk_mq_alloc_sched_res(q, &ctx->res,
-                                       nr_hw_queues);
+                       ret = blk_mq_alloc_sched_res(q, q->elevator->type,
+                                       &ctx->res, nr_hw_queues);
                        if (ret)
                                goto out_unwind;
                }
        }
        return 0;
+
 out_unwind:
        list_for_each_entry_continue_reverse(q, &set->tag_list, tag_set_list) {
                if (q->elevator) {
                        ctx = xa_load(elv_tbl, q->id);
                        if (ctx)
-                               blk_mq_free_sched_res(&ctx->res, set);
+                               blk_mq_free_sched_res(&ctx->res,
+                                               ctx->type, set);
                }
        }
        return ret;
@@ -605,7 +620,7 @@ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e,
        unsigned long i;
        int ret;
 
-       eq = elevator_alloc(q, e, et);
+       eq = elevator_alloc(q, e, res);
        if (!eq)
                return -ENOMEM;
 

diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h
index 4e1b86e85a8a9754a62aec907df41cc8634c4702..02c40a72e9598c7ff6be2b175eba0cc4587a378c 100644 (file)
--- a/block/blk-mq-sched.h
+++ b/block/blk-mq-sched.h
@@ -26,7 +26,9 @@ void blk_mq_sched_free_rqs(struct request_queue *q);
 struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set,
                unsigned int nr_hw_queues, unsigned int nr_requests);
 int blk_mq_alloc_sched_res(struct request_queue *q,
-               struct elevator_resources *res, unsigned int nr_hw_queues);
+               struct elevator_type *type,
+               struct elevator_resources *res,
+               unsigned int nr_hw_queues);
 int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl,
                struct blk_mq_tag_set *set, unsigned int nr_hw_queues);
 int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl,
@@ -35,6 +37,7 @@ void blk_mq_free_sched_ctx_batch(struct xarray *elv_tbl);
 void blk_mq_free_sched_tags(struct elevator_tags *et,
                struct blk_mq_tag_set *set);
 void blk_mq_free_sched_res(struct elevator_resources *res,
+               struct elevator_type *type,
                struct blk_mq_tag_set *set);
 void blk_mq_free_sched_res_batch(struct xarray *et_table,
                struct blk_mq_tag_set *set);

diff --git a/block/elevator.c b/block/elevator.c
index cbec292a4af5b123c8f274bb54754737c14f913a..5b37ef44f52d7f568fd2b5292b07864253ca41e6 100644 (file)
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -121,7 +121,7 @@ static struct elevator_type *elevator_find_get(const char *name)
 static const struct kobj_type elv_ktype;
 
 struct elevator_queue *elevator_alloc(struct request_queue *q,
-               struct elevator_type *e, struct elevator_tags *et)
+               struct elevator_type *e, struct elevator_resources *res)
 {
        struct elevator_queue *eq;
 
@@ -134,7 +134,8 @@ struct elevator_queue *elevator_alloc(struct request_queue *q,
        kobject_init(&eq->kobj, &elv_ktype);
        mutex_init(&eq->sysfs_lock);
        hash_init(eq->hash);
-       eq->et = et;
+       eq->et = res->et;
+       eq->elevator_data = res->data;
 
        return eq;
 }
@@ -617,7 +618,7 @@ static void elv_exit_and_release(struct elv_change_ctx *ctx,
        mutex_unlock(&q->elevator_lock);
        blk_mq_unfreeze_queue(q, memflags);
        if (e) {
-               blk_mq_free_sched_res(&ctx->res, q->tag_set);
+               blk_mq_free_sched_res(&ctx->res, ctx->type, q->tag_set);
                kobject_put(&e->kobj);
        }
 }
@@ -628,12 +629,15 @@ static int elevator_change_done(struct request_queue *q,
        int ret = 0;
 
        if (ctx->old) {
-               struct elevator_resources res = {.et = ctx->old->et};
+               struct elevator_resources res = {
+                       .et = ctx->old->et,
+                       .data = ctx->old->elevator_data
+               };
                bool enable_wbt = test_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT,
                                &ctx->old->flags);
 
                elv_unregister_queue(q, ctx->old);
-               blk_mq_free_sched_res(&res, q->tag_set);
+               blk_mq_free_sched_res(&res, ctx->old->type, q->tag_set);
                kobject_put(&ctx->old->kobj);
                if (enable_wbt)
                        wbt_enable_default(q->disk);
@@ -658,7 +662,8 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx)
        lockdep_assert_held(&set->update_nr_hwq_lock);
 
        if (strncmp(ctx->name, "none", 4)) {
-               ret = blk_mq_alloc_sched_res(q, &ctx->res, set->nr_hw_queues);
+               ret = blk_mq_alloc_sched_res(q, ctx->type, &ctx->res,
+                               set->nr_hw_queues);
                if (ret)
                        return ret;
        }
@@ -681,11 +686,12 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx)
        blk_mq_unfreeze_queue(q, memflags);
        if (!ret)
                ret = elevator_change_done(q, ctx);
+
        /*
         * Free sched resource if it's allocated but we couldn't switch elevator.
         */
        if (!ctx->new)
-               blk_mq_free_sched_res(&ctx->res, set);
+               blk_mq_free_sched_res(&ctx->res, ctx->type, set);
 
        return ret;
 }
@@ -711,11 +717,12 @@ void elv_update_nr_hw_queues(struct request_queue *q,
        blk_mq_unfreeze_queue_nomemrestore(q);
        if (!ret)
                WARN_ON_ONCE(elevator_change_done(q, ctx));
+
        /*
         * Free sched resource if it's allocated but we couldn't switch elevator.
         */
        if (!ctx->new)
-               blk_mq_free_sched_res(&ctx->res, set);
+               blk_mq_free_sched_res(&ctx->res, ctx->type, set);
 }
 
 /*
@@ -729,7 +736,6 @@ void elevator_set_default(struct request_queue *q)
                .no_uevent = true,
        };
        int err;
-       struct elevator_type *e;
 
        /* now we allow to switch elevator */
        blk_queue_flag_clear(QUEUE_FLAG_NO_ELV_SWITCH, q);
@@ -742,8 +748,8 @@ void elevator_set_default(struct request_queue *q)
         * have multiple queues or mq-deadline is not available, default
         * to "none".
         */
-       e = elevator_find_get(ctx.name);
-       if (!e)
+       ctx.type = elevator_find_get(ctx.name);
+       if (!ctx.type)
                return;
 
        if ((q->nr_hw_queues == 1 ||
@@ -753,7 +759,7 @@ void elevator_set_default(struct request_queue *q)
                        pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n",
                                        ctx.name, err);
        }
-       elevator_put(e);
+       elevator_put(ctx.type);
 }
 
 void elevator_set_none(struct request_queue *q)
@@ -802,6 +808,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
        ctx.name = strstrip(elevator_name);
 
        elv_iosched_load_module(ctx.name);
+       ctx.type = elevator_find_get(ctx.name);
 
        down_read(&set->update_nr_hwq_lock);
        if (!blk_queue_no_elv_switch(q)) {
@@ -812,6 +819,9 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
                ret = -ENOENT;
        }
        up_read(&set->update_nr_hwq_lock);
+
+       if (ctx.type)
+               elevator_put(ctx.type);
        return ret;
 }
 

diff --git a/block/elevator.h b/block/elevator.h
index e34043f6da26e3010cf25ad88610b070a3705a81..3ee1d494f48afc3b5f9e05f634aca30d4566551d 100644 (file)
--- a/block/elevator.h
+++ b/block/elevator.h
@@ -33,6 +33,8 @@ struct elevator_tags {
 };
 
 struct elevator_resources {
+       /* holds elevator data */
+       void *data;
        /* holds elevator tags */
        struct elevator_tags *et;
 };
@@ -185,7 +187,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *page, size_t count);
 
 extern bool elv_bio_merge_ok(struct request *, struct bio *);
 struct elevator_queue *elevator_alloc(struct request_queue *,
-               struct elevator_type *, struct elevator_tags *);
+               struct elevator_type *, struct elevator_resources *);
 
 /*
  * Helper functions.