blk-cgroup: don't nest queue_lock under rcu in blkg_lookup_create()

Change this in two steps:

1) hold rcu lock and do blkg_lookup() from fast path;
2) hold queue_lock directly from slow path, and don't nest it under rcu
   lock;

Prepare to convert protecting blkcg with blkcg_mutex instead of
queue_lock.

Signed-off-by: Yu Kuai <yukuai@fygo.io>
Link: https://patch.msgid.link/93f33cc9e5a39dddb78dcd934d0c1d04b564fb00.1780621988.git.yukuai@fygo.io
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index d6355338f2900b3d08ba430ca6864d143cdf9762..fee8c9d5dc2c0fb21ac040db56d1b376733b0983 100644 (file)
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -469,22 +469,17 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
 {
        struct request_queue *q = disk->queue;
        struct blkcg_gq *blkg;
-       unsigned long flags;
-
-       WARN_ON_ONCE(!rcu_read_lock_held());
 
-       blkg = blkg_lookup(blkcg, q);
-       if (blkg)
-               return blkg;
-
-       spin_lock_irqsave(&q->queue_lock, flags);
+       rcu_read_lock();
        blkg = blkg_lookup(blkcg, q);
        if (blkg) {
                if (blkcg != &blkcg_root &&
                    blkg != rcu_dereference(blkcg->blkg_hint))
                        rcu_assign_pointer(blkcg->blkg_hint, blkg);
-               goto found;
+               rcu_read_unlock();
+               return blkg;
        }
+       rcu_read_unlock();
 
        /*
         * Create blkgs walking down from blkcg_root to @blkcg, so that all
@@ -516,8 +511,6 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
                        break;
        }
 
-found:
-       spin_unlock_irqrestore(&q->queue_lock, flags);
        return blkg;
 }
 
@@ -2046,6 +2039,18 @@ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
        atomic64_add(delta, &blkg->delay_nsec);
 }
 
+static inline struct blkcg_gq *blkg_lookup_tryget(struct blkcg_gq *blkg)
+{
+retry:
+       if (blkg_tryget(blkg))
+               return blkg;
+
+       blkg = blkg->parent;
+       if (blkg)
+               goto retry;
+
+       return NULL;
+}
 /**
  * blkg_tryget_closest - try and get a blkg ref on the closet blkg
  * @bio: target bio
@@ -2058,20 +2063,30 @@ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
 static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
                struct cgroup_subsys_state *css)
 {
-       struct blkcg_gq *blkg, *ret_blkg = NULL;
+       struct request_queue *q = bio->bi_bdev->bd_queue;
+       struct blkcg *blkcg = css_to_blkcg(css);
+       struct blkcg_gq *blkg;
 
        rcu_read_lock();
-       blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk);
-       while (blkg) {
-               if (blkg_tryget(blkg)) {
-                       ret_blkg = blkg;
-                       break;
-               }
-               blkg = blkg->parent;
-       }
+       blkg = blkg_lookup(blkcg, q);
+       if (likely(blkg))
+               blkg = blkg_lookup_tryget(blkg);
        rcu_read_unlock();
 
-       return ret_blkg;
+       if (blkg)
+               return blkg;
+
+       /*
+        * Fast path failed, we're probably issuing IO in this cgroup the first
+        * time, hold lock to create new blkg.
+        */
+       spin_lock_irq(&q->queue_lock);
+       blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk);
+       if (blkg)
+               blkg = blkg_lookup_tryget(blkg);
+       spin_unlock_irq(&q->queue_lock);
+
+       return blkg;
 }
 
 /**