block: move wbt_enable_default() out of queue freezing from sched ->exit()

scheduler's ->exit() is called with queue frozen and elevator lock is held, and
wbt_enable_default() can't be called with queue frozen, otherwise the
following lockdep warning is triggered:

	#6 (&q->rq_qos_mutex){+.+.}-{4:4}:
	#5 (&eq->sysfs_lock){+.+.}-{4:4}:
	#4 (&q->elevator_lock){+.+.}-{4:4}:
	#3 (&q->q_usage_counter(io)#3){++++}-{0:0}:
	#2 (fs_reclaim){+.+.}-{0:0}:
	#1 (&sb->s_type->i_mutex_key#3){+.+.}-{4:4}:
	#0 (&q->debugfs_mutex){+.+.}-{4:4}:

Fix the issue by moving wbt_enable_default() out of bfq's exit(), and
call it from elevator_change_done().

Meantime add disk->rqos_state_mutex for covering wbt state change, which
matches the purpose more than ->elevator_lock.

Reviewed-by: Hannes Reinecke <hare@suse.de>
Reviewed-by: Nilay Shroff <nilay@linux.ibm.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/20250505141805.2751237-26-ming.lei@redhat.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index cc6f59836dcd2b40635935f32b87cee57e62a7d0..0cb1e9873aabb2d3d61a25970e003b965893335e 100644 (file)
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -7211,7 +7211,7 @@ static void bfq_exit_queue(struct elevator_queue *e)
 
        blk_stat_disable_accounting(bfqd->queue);
        blk_queue_flag_clear(QUEUE_FLAG_DISABLE_WBT_DEF, bfqd->queue);
-       wbt_enable_default(bfqd->queue->disk);
+       set_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT, &e->flags);
 
        kfree(bfqd);
 }

diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 741e607dfab6f1f005a7b7673f9b6b45b5bb23d8..01e0ead1327871fe84518d61e9aff64e6b796fb1 100644 (file)
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -560,7 +560,7 @@ static ssize_t queue_wb_lat_show(struct gendisk *disk, char *page)
        ssize_t ret;
        struct request_queue *q = disk->queue;
 
-       mutex_lock(&q->elevator_lock);
+       mutex_lock(&disk->rqos_state_mutex);
        if (!wbt_rq_qos(q)) {
                ret = -EINVAL;
                goto out;
@@ -573,7 +573,7 @@ static ssize_t queue_wb_lat_show(struct gendisk *disk, char *page)
 
        ret = sysfs_emit(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000));
 out:
-       mutex_unlock(&q->elevator_lock);
+       mutex_unlock(&disk->rqos_state_mutex);
        return ret;
 }
 
@@ -593,7 +593,6 @@ static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page,
                return -EINVAL;
 
        memflags = blk_mq_freeze_queue(q);
-       mutex_lock(&q->elevator_lock);
 
        rqos = wbt_rq_qos(q);
        if (!rqos) {
@@ -618,11 +617,12 @@ static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page,
         */
        blk_mq_quiesce_queue(q);
 
+       mutex_lock(&disk->rqos_state_mutex);
        wbt_set_min_lat(q, val);
+       mutex_unlock(&disk->rqos_state_mutex);
 
        blk_mq_unquiesce_queue(q);
 out:
-       mutex_unlock(&q->elevator_lock);
        blk_mq_unfreeze_queue(q, memflags);
 
        return ret;
@@ -871,9 +871,7 @@ int blk_register_queue(struct gendisk *disk)
 
        if (queue_is_mq(q))
                elevator_set_default(q);
-       mutex_lock(&q->elevator_lock);
        wbt_enable_default(disk);
-       mutex_unlock(&q->elevator_lock);
 
        blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
 

diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index 29cd2e33666fec9e871ce3dde549a64f7de042ff..74ae7131ada99ada3a4e1857f8ced61c542f0b7a 100644 (file)
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -704,6 +704,8 @@ void wbt_enable_default(struct gendisk *disk)
        struct rq_qos *rqos;
        bool enable = IS_ENABLED(CONFIG_BLK_WBT_MQ);
 
+       mutex_lock(&disk->rqos_state_mutex);
+
        if (blk_queue_disable_wbt(q))
                enable = false;
 
@@ -712,8 +714,10 @@ void wbt_enable_default(struct gendisk *disk)
        if (rqos) {
                if (enable && RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT)
                        RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT;
+               mutex_unlock(&disk->rqos_state_mutex);
                return;
        }
+       mutex_unlock(&disk->rqos_state_mutex);
 
        /* Queue not registered? Maybe shutting down... */
        if (!blk_queue_registered(q))
@@ -773,11 +777,13 @@ void wbt_disable_default(struct gendisk *disk)
        struct rq_wb *rwb;
        if (!rqos)
                return;
+       mutex_lock(&disk->rqos_state_mutex);
        rwb = RQWB(rqos);
        if (rwb->enable_state == WBT_STATE_ON_DEFAULT) {
                blk_stat_deactivate(rwb->cb);
                rwb->enable_state = WBT_STATE_OFF_DEFAULT;
        }
+       mutex_unlock(&disk->rqos_state_mutex);
 }
 EXPORT_SYMBOL_GPL(wbt_disable_default);
 

diff --git a/block/elevator.c b/block/elevator.c
index 8578b969e1733aade9096304569147a3137d8e8b..f8d72bd206103090d2acc5d620267621b30d388f 100644 (file)
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -637,8 +637,13 @@ static int elevator_change_done(struct request_queue *q,
        int ret = 0;
 
        if (ctx->old) {
+               bool enable_wbt = test_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT,
+                               &ctx->old->flags);
+
                elv_unregister_queue(q, ctx->old);
                kobject_put(&ctx->old->kobj);
+               if (enable_wbt)
+                       wbt_enable_default(q->disk);
        }
        if (ctx->new) {
                ret = elv_register_queue(q, ctx->new, !ctx->no_uevent);

diff --git a/block/elevator.h b/block/elevator.h
index 76a90a1b7ed6559e328deae8ec3f43a9e3364e35..a07ce773a38f799cf9c9b8e56c0599a0fa4c9cb6 100644 (file)
--- a/block/elevator.h
+++ b/block/elevator.h
@@ -122,6 +122,7 @@ struct elevator_queue
 
 #define ELEVATOR_FLAG_REGISTERED       0
 #define ELEVATOR_FLAG_DYING            1
+#define ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT       2
 
 /*
  * block elevator interface

diff --git a/block/genhd.c b/block/genhd.c
index a8cb5607b6e3d303288b5505a1d122a6010e16b8..9c7c657380db75e5ad61940a0b15a0f7d7a90c9f 100644 (file)
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1470,6 +1470,7 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
        INIT_LIST_HEAD(&disk->slave_bdevs);
 #endif
+       mutex_init(&disk->rqos_state_mutex);
        return disk;
 
 out_erase_part0:

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 3aa1fd637d5774d80949a38a7eb38b0925434ea7..94323f303b37dc9da0d1590349a4e86ff89a4220 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -218,6 +218,8 @@ struct gendisk {
         * devices that do not have multiple independent access ranges.
         */
        struct blk_independent_access_ranges *ia_ranges;
+
+       struct mutex rqos_state_mutex;  /* rqos state change mutex */
 };
 
 /**