blk-cgroup: delay freeing policy data after rcu grace period

Currently blkcg_print_blkgs() must hold RCU to iterate blkgs from a
blkcg, and prfill() must hold queue_lock to prevent policy data from
being freed by policy deactivation. As a consequence, queue_lock has to
be nested under RCU from blkcg_print_blkgs().

Delay freeing policy data until after an RCU grace period so prfill() can
be protected by RCU alone.

Signed-off-by: Yu Kuai <yukuai@fygo.io>
Link: https://patch.msgid.link/e20e5d984b41a026d61851966bed35eb094c4bff.1780621988.git.yukuai@fygo.io
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index d8fdace464b450040416c9a2566eca0b9235d5b3..0d3e32d246a24e33ded7185724be2eae1224b388 100644 (file)
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -550,13 +550,20 @@ static void bfq_pd_init(struct blkg_policy_data *pd)
        bfqg->rq_pos_tree = RB_ROOT;
 }
 
-static void bfq_pd_free(struct blkg_policy_data *pd)
+static void bfqg_release(struct rcu_head *rcu)
 {
+       struct blkg_policy_data *pd =
+               container_of(rcu, struct blkg_policy_data, rcu_head);
        struct bfq_group *bfqg = pd_to_bfqg(pd);
 
        bfqg_put(bfqg);
 }
 
+static void bfq_pd_free(struct blkg_policy_data *pd)
+{
+       call_rcu(&pd->rcu_head, bfqg_release);
+}
+
 static void bfq_pd_reset_stats(struct blkg_policy_data *pd)
 {
        struct bfq_group *bfqg = pd_to_bfqg(pd);

diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h
index f25fecb87c43f2a9d5238f7ce54e22481fe78d28..cc603ded6ded43301c8e4b53f0b8bc28f6cabcda 100644 (file)
--- a/block/blk-cgroup.h
+++ b/block/blk-cgroup.h
@@ -140,6 +140,8 @@ struct blkg_policy_data {
        struct blkcg_gq                 *blkg;
        int                             plid;
        bool                            online;
+
+       struct rcu_head                 rcu_head;
 };
 
 /*

diff --git a/block/blk-iocost.c b/block/blk-iocost.c
index 563cc7dcf348035e777766ebcea63598e1abe827..27d2dcaa65f02694c7a990cfb53b02f8343a324c 100644 (file)
--- a/block/blk-iocost.c
+++ b/block/blk-iocost.c
@@ -3050,6 +3050,16 @@ static void ioc_pd_init(struct blkg_policy_data *pd)
        spin_unlock_irqrestore(&ioc->lock, flags);
 }
 
+static void iocg_release(struct rcu_head *rcu)
+{
+       struct blkg_policy_data *pd =
+               container_of(rcu, struct blkg_policy_data, rcu_head);
+       struct ioc_gq *iocg = pd_to_iocg(pd);
+
+       free_percpu(iocg->pcpu_stat);
+       kfree(iocg);
+}
+
 static void ioc_pd_free(struct blkg_policy_data *pd)
 {
        struct ioc_gq *iocg = pd_to_iocg(pd);
@@ -3074,8 +3084,8 @@ static void ioc_pd_free(struct blkg_policy_data *pd)
 
                hrtimer_cancel(&iocg->waitq_timer);
        }
-       free_percpu(iocg->pcpu_stat);
-       kfree(iocg);
+
+       call_rcu(&pd->rcu_head, iocg_release);
 }
 
 static void ioc_pd_stat(struct blkg_policy_data *pd, struct seq_file *s)

diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c
index 1aaee6fb0f59ff2cd0c89e7b015d72ad8c5c05dc..cef02b6c5fa91d5922eb593460c01733d28804ed 100644 (file)
--- a/block/blk-iolatency.c
+++ b/block/blk-iolatency.c
@@ -1031,13 +1031,21 @@ static void iolatency_pd_offline(struct blkg_policy_data *pd)
        iolatency_clear_scaling(blkg);
 }
 
-static void iolatency_pd_free(struct blkg_policy_data *pd)
+static void iolat_release(struct rcu_head *rcu)
 {
+       struct blkg_policy_data *pd =
+               container_of(rcu, struct blkg_policy_data, rcu_head);
        struct iolatency_grp *iolat = pd_to_lat(pd);
+
        free_percpu(iolat->stats);
        kfree(iolat);
 }
 
+static void iolatency_pd_free(struct blkg_policy_data *pd)
+{
+       call_rcu(&pd->rcu_head, iolat_release);
+}
+
 static struct cftype iolatency_files[] = {
        {
                .name = "latency",

diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 47052ba21d1bcc87a02a8a61a76a2a081be97d6c..ffc3b70065d4b1ec8dfa254686a3b30beb04253d 100644 (file)
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -353,16 +353,25 @@ static void throtl_pd_online(struct blkg_policy_data *pd)
        tg_update_has_rules(tg);
 }
 
-static void throtl_pd_free(struct blkg_policy_data *pd)
+static void tg_release(struct rcu_head *rcu)
 {
+       struct blkg_policy_data *pd =
+               container_of(rcu, struct blkg_policy_data, rcu_head);
        struct throtl_grp *tg = pd_to_tg(pd);
 
-       timer_delete_sync(&tg->service_queue.pending_timer);
        blkg_rwstat_exit(&tg->stat_bytes);
        blkg_rwstat_exit(&tg->stat_ios);
        kfree(tg);
 }
 
+static void throtl_pd_free(struct blkg_policy_data *pd)
+{
+       struct throtl_grp *tg = pd_to_tg(pd);
+
+       timer_delete_sync(&tg->service_queue.pending_timer);
+       call_rcu(&pd->rcu_head, tg_release);
+}
+
 static struct throtl_grp *
 throtl_rb_first(struct throtl_service_queue *parent_sq)
 {