bcachefs: Refactor open_bucket code

Prep work for adding a hash table of open buckets - instead of embedding
a bch_extent_ptr, we need to refer to the bucket directly so that we're
not calling sector_to_bucket() in the hash table lookup code, which has
an expensive divide.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>

diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c
index fe7bc3cdee300207bdda46bdc79be77990df6910..8831b2a0303a85fc9156a321fa874494b80d978d 100644 (file)
--- a/fs/bcachefs/alloc_background.c
+++ b/fs/bcachefs/alloc_background.c
@@ -1066,7 +1066,7 @@ static bool bch2_dev_has_open_write_point(struct bch_fs *c, struct bch_dev *ca)
             ob++) {
                spin_lock(&ob->lock);
                if (ob->valid && !ob->on_partial_list &&
-                   ob->ptr.dev == ca->dev_idx)
+                   ob->dev == ca->dev_idx)
                        ret = true;
                spin_unlock(&ob->lock);
        }
@@ -1213,22 +1213,3 @@ void bch2_fs_allocator_background_init(struct bch_fs *c)
 {
        spin_lock_init(&c->freelist_lock);
 }
-
-void bch2_open_buckets_to_text(struct printbuf *out, struct bch_fs *c)
-{
-       struct open_bucket *ob;
-
-       for (ob = c->open_buckets;
-            ob < c->open_buckets + ARRAY_SIZE(c->open_buckets);
-            ob++) {
-               spin_lock(&ob->lock);
-               if (ob->valid && !ob->on_partial_list) {
-                       pr_buf(out, "%zu ref %u type %s\n",
-                              ob - c->open_buckets,
-                              atomic_read(&ob->pin),
-                              bch2_data_types[ob->type]);
-               }
-               spin_unlock(&ob->lock);
-       }
-
-}

diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h
index e3cdb8bc1dd8f6ef260166e88fbf2f4559de6bf3..86b64177b3d0bc2a378e7961aff478363ce0c210 100644 (file)
--- a/fs/bcachefs/alloc_background.h
+++ b/fs/bcachefs/alloc_background.h
@@ -142,6 +142,4 @@ int bch2_dev_allocator_start(struct bch_dev *);
 int bch2_alloc_write_all(struct bch_fs *, unsigned);
 void bch2_fs_allocator_background_init(struct bch_fs *);
 
-void bch2_open_buckets_to_text(struct printbuf *, struct bch_fs *);
-
 #endif /* _BCACHEFS_ALLOC_BACKGROUND_H */

diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c
index af651dd9a36f34799b0a73f266adcbd45dc12f94..35a2683d88076a3fceb476a3eef0e16c9162c441 100644 (file)
--- a/fs/bcachefs/alloc_foreground.c
+++ b/fs/bcachefs/alloc_foreground.c
@@ -45,7 +45,7 @@
 
 void __bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob)
 {
-       struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
+       struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
 
        if (ob->ec) {
                bch2_ec_bucket_written(c, ob);
@@ -55,9 +55,9 @@ void __bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob)
        percpu_down_read(&c->mark_lock);
        spin_lock(&ob->lock);
 
-       bch2_mark_alloc_bucket(c, ca, PTR_BUCKET_NR(ca, &ob->ptr), false);
+       bch2_mark_alloc_bucket(c, ca, ob->bucket, false);
        ob->valid = false;
-       ob->type = 0;
+       ob->data_type = 0;
 
        spin_unlock(&ob->lock);
        percpu_up_read(&c->mark_lock);
@@ -81,8 +81,7 @@ void bch2_open_bucket_write_error(struct bch_fs *c,
        unsigned i;
 
        open_bucket_for_each(c, obs, ob, i)
-               if (ob->ptr.dev == dev &&
-                   ob->ec)
+               if (ob->dev == dev && ob->ec)
                        bch2_ec_bucket_cancel(c, ob);
 }
 
@@ -95,18 +94,19 @@ static struct open_bucket *bch2_open_bucket_alloc(struct bch_fs *c)
        ob = c->open_buckets + c->open_buckets_freelist;
        c->open_buckets_freelist = ob->freelist;
        atomic_set(&ob->pin, 1);
-       ob->type = 0;
+       ob->data_type = 0;
 
        c->open_buckets_nr_free--;
        return ob;
 }
 
+
 static void open_bucket_free_unused(struct bch_fs *c,
                                    struct write_point *wp,
                                    struct open_bucket *ob)
 {
-       struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
-       bool may_realloc = wp->type == BCH_DATA_user;
+       struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
+       bool may_realloc = wp->data_type == BCH_DATA_user;
 
        BUG_ON(ca->open_buckets_partial_nr >
               ARRAY_SIZE(ca->open_buckets_partial));
@@ -133,11 +133,13 @@ static void verify_not_stale(struct bch_fs *c, const struct open_buckets *obs)
        struct open_bucket *ob;
        unsigned i;
 
+       rcu_read_lock();
        open_bucket_for_each(c, obs, ob, i) {
-               struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
+               struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
 
-               BUG_ON(ptr_stale(ca, &ob->ptr));
+               BUG_ON(bucket(ca, ob->bucket)->mark.gen != ob->gen);
        }
+       rcu_read_unlock();
 #endif
 }
 
@@ -246,13 +248,9 @@ out:
        ob->valid       = true;
        ob->sectors_free = ca->mi.bucket_size;
        ob->alloc_reserve = reserve;
-       ob->ptr         = (struct bch_extent_ptr) {
-               .type   = 1 << BCH_EXTENT_ENTRY_ptr,
-               .gen    = bucket(ca, b)->mark.gen,
-               .offset = bucket_to_sector(ca, b),
-               .dev    = ca->dev_idx,
-       };
-
+       ob->dev         = ca->dev_idx;
+       ob->gen         = bucket(ca, b)->mark.gen;
+       ob->bucket      = b;
        spin_unlock(&ob->lock);
 
        if (c->blocked_allocate_open_bucket) {
@@ -333,9 +331,9 @@ static void add_new_bucket(struct bch_fs *c,
                           struct open_bucket *ob)
 {
        unsigned durability =
-               bch_dev_bkey_exists(c, ob->ptr.dev)->mi.durability;
+               bch_dev_bkey_exists(c, ob->dev)->mi.durability;
 
-       __clear_bit(ob->ptr.dev, devs_may_alloc->d);
+       __clear_bit(ob->dev, devs_may_alloc->d);
        *nr_effective   += (flags & BUCKET_ALLOC_USE_DURABILITY)
                ? durability : 1;
        *have_cache     |= !durability;
@@ -445,13 +443,13 @@ static int bucket_alloc_from_stripe(struct bch_fs *c,
                                continue;
 
                        ob = c->open_buckets + h->s->blocks[ec_idx];
-                       if (ob->ptr.dev == devs_sorted.devs[i] &&
+                       if (ob->dev == devs_sorted.devs[i] &&
                            !test_and_set_bit(ec_idx, h->s->blocks_allocated))
                                goto got_bucket;
                }
        goto out_put_head;
 got_bucket:
-       ca = bch_dev_bkey_exists(c, ob->ptr.dev);
+       ca = bch_dev_bkey_exists(c, ob->dev);
 
        ob->ec_idx      = ec_idx;
        ob->ec          = h->s;
@@ -481,12 +479,12 @@ static void get_buckets_from_writepoint(struct bch_fs *c,
        unsigned i;
 
        open_bucket_for_each(c, &wp->ptrs, ob, i) {
-               struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
+               struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
 
                if (*nr_effective < nr_replicas &&
-                   test_bit(ob->ptr.dev, devs_may_alloc->d) &&
+                   test_bit(ob->dev, devs_may_alloc->d) &&
                    (ca->mi.durability ||
-                    (wp->type == BCH_DATA_user && !*have_cache)) &&
+                    (wp->data_type == BCH_DATA_user && !*have_cache)) &&
                    (ob->ec || !need_ec)) {
                        add_new_bucket(c, ptrs, devs_may_alloc,
                                       nr_effective, have_cache,
@@ -518,7 +516,7 @@ static int open_bucket_add_buckets(struct bch_fs *c,
        unsigned i;
 
        rcu_read_lock();
-       devs = target_rw_devs(c, wp->type, target);
+       devs = target_rw_devs(c, wp->data_type, target);
        rcu_read_unlock();
 
        /* Don't allocate from devices we already have pointers to: */
@@ -526,7 +524,7 @@ static int open_bucket_add_buckets(struct bch_fs *c,
                __clear_bit(devs_have->devs[i], devs.d);
 
        open_bucket_for_each(c, ptrs, ob, i)
-               __clear_bit(ob->ptr.dev, devs.d);
+               __clear_bit(ob->dev, devs.d);
 
        if (erasure_code) {
                if (!ec_open_bucket(c, ptrs)) {
@@ -586,7 +584,7 @@ void bch2_open_buckets_stop_dev(struct bch_fs *c, struct bch_dev *ca,
        unsigned i, j;
 
        open_bucket_for_each(c, obs, ob, i) {
-               bool drop = !ca || ob->ptr.dev == ca->dev_idx;
+               bool drop = !ca || ob->dev == ca->dev_idx;
 
                if (!drop && ob->ec) {
                        mutex_lock(&ob->ec->lock);
@@ -595,7 +593,7 @@ void bch2_open_buckets_stop_dev(struct bch_fs *c, struct bch_dev *ca,
                                        continue;
 
                                ob2 = c->open_buckets + ob->ec->blocks[j];
-                               drop |= ob2->ptr.dev == ca->dev_idx;
+                               drop |= ob2->dev == ca->dev_idx;
                        }
                        mutex_unlock(&ob->ec->lock);
                }
@@ -779,11 +777,11 @@ retry:
 
        wp = writepoint_find(c, write_point.v);
 
-       if (wp->type == BCH_DATA_user)
+       if (wp->data_type == BCH_DATA_user)
                ob_flags |= BUCKET_MAY_ALLOC_PARTIAL;
 
        /* metadata may not allocate on cache devices: */
-       if (wp->type != BCH_DATA_user)
+       if (wp->data_type != BCH_DATA_user)
                have_cache = true;
 
        if (!target || (flags & BCH_WRITE_ONLY_SPECIFIED_DEVS)) {
@@ -861,6 +859,20 @@ err:
        }
 }
 
+struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *c, struct open_bucket *ob)
+{
+       struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
+
+       return (struct bch_extent_ptr) {
+               .type   = 1 << BCH_EXTENT_ENTRY_ptr,
+               .gen    = ob->gen,
+               .dev    = ob->dev,
+               .offset = bucket_to_sector(ca, ob->bucket) +
+                       ca->mi.bucket_size -
+                       ob->sectors_free,
+       };
+}
+
 /*
  * Append pointers to the space we just allocated to @k, and mark @sectors space
  * as allocated out of @ob
@@ -877,14 +889,13 @@ void bch2_alloc_sectors_append_ptrs(struct bch_fs *c, struct write_point *wp,
        wp->sectors_free -= sectors;
 
        open_bucket_for_each(c, &wp->ptrs, ob, i) {
-               struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
-               struct bch_extent_ptr ptr = ob->ptr;
+               struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
+               struct bch_extent_ptr ptr = bch2_ob_ptr(c, ob);
 
                ptr.cached = cached ||
                        (!ca->mi.durability &&
-                        wp->type == BCH_DATA_user);
+                        wp->data_type == BCH_DATA_user);
 
-               ptr.offset += ca->mi.bucket_size - ob->sectors_free;
                bch2_bkey_append_ptr(k, ptr);
 
                BUG_ON(sectors > ob->sectors_free);
@@ -915,7 +926,7 @@ static inline void writepoint_init(struct write_point *wp,
                                   enum bch_data_type type)
 {
        mutex_init(&wp->lock);
-       wp->type = type;
+       wp->data_type = type;
 }
 
 void bch2_fs_allocator_foreground_init(struct bch_fs *c)
@@ -952,3 +963,22 @@ void bch2_fs_allocator_foreground_init(struct bch_fs *c)
                                   writepoint_hash(c, wp->write_point));
        }
 }
+
+void bch2_open_buckets_to_text(struct printbuf *out, struct bch_fs *c)
+{
+       struct open_bucket *ob;
+
+       for (ob = c->open_buckets;
+            ob < c->open_buckets + ARRAY_SIZE(c->open_buckets);
+            ob++) {
+               spin_lock(&ob->lock);
+               if (ob->valid && !ob->on_partial_list) {
+                       pr_buf(out, "%zu ref %u type %s\n",
+                              ob - c->open_buckets,
+                              atomic_read(&ob->pin),
+                              bch2_data_types[ob->data_type]);
+               }
+               spin_unlock(&ob->lock);
+       }
+
+}

diff --git a/fs/bcachefs/alloc_foreground.h b/fs/bcachefs/alloc_foreground.h
index d8888785676d85c6c7b4944462c046895b28e731..39d8ae5bbb96921384261ca6c476fce1baa3a1b5 100644 (file)
--- a/fs/bcachefs/alloc_foreground.h
+++ b/fs/bcachefs/alloc_foreground.h
@@ -85,7 +85,7 @@ static inline void bch2_open_bucket_get(struct bch_fs *c,
        unsigned i;
 
        open_bucket_for_each(c, &wp->ptrs, ob, i) {
-               ob->type = wp->type;
+               ob->data_type = wp->data_type;
                atomic_inc(&ob->pin);
                ob_push(c, ptrs, ob);
        }
@@ -105,6 +105,7 @@ struct write_point *bch2_alloc_sectors_start(struct bch_fs *,
                                             unsigned,
                                             struct closure *);
 
+struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *, struct open_bucket *);
 void bch2_alloc_sectors_append_ptrs(struct bch_fs *, struct write_point *,
                                    struct bkey_i *, unsigned, bool);
 void bch2_alloc_sectors_done(struct bch_fs *, struct write_point *);
@@ -127,4 +128,6 @@ static inline struct write_point_specifier writepoint_ptr(struct write_point *wp
 
 void bch2_fs_allocator_foreground_init(struct bch_fs *);
 
+void bch2_open_buckets_to_text(struct printbuf *, struct bch_fs *);
+
 #endif /* _BCACHEFS_ALLOC_FOREGROUND_H */

diff --git a/fs/bcachefs/alloc_types.h b/fs/bcachefs/alloc_types.h
index 4a1cd8b73d16b1e500a84ae9290d9bf144c502cf..bd173c7c334b462126a8cd176b400887cc059de9 100644 (file)
--- a/fs/bcachefs/alloc_types.h
+++ b/fs/bcachefs/alloc_types.h
@@ -49,12 +49,15 @@ struct open_bucket {
         * the block in the stripe this open_bucket corresponds to:
         */
        u8                      ec_idx;
-       u8                      type;
+       enum bch_data_type      data_type:3;
        unsigned                valid:1;
        unsigned                on_partial_list:1;
        int                     alloc_reserve:3;
+
        unsigned                sectors_free;
-       struct bch_extent_ptr   ptr;
+       u8                      dev;
+       u8                      gen;
+       u64                     bucket;
        struct ec_stripe_new    *ec;
 };
 
@@ -74,7 +77,7 @@ struct write_point {
        struct mutex            lock;
        u64                     last_used;
        unsigned long           write_point;
-       enum bch_data_type      type;
+       enum bch_data_type      data_type;
 
        /* calculated based on how many pointers we're actually going to use: */
        unsigned                sectors_free;

diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c
index c3f86cc39842ce6f52f044e6d2478d2d23b7bbd0..05f55b74d6413c79d1429dc628e3aca2a5b97002 100644 (file)
--- a/fs/bcachefs/ec.c
+++ b/fs/bcachefs/ec.c
@@ -1063,7 +1063,7 @@ void *bch2_writepoint_ec_buf(struct bch_fs *c, struct write_point *wp)
        if (!ob)
                return NULL;
 
-       ca      = bch_dev_bkey_exists(c, ob->ptr.dev);
+       ca      = bch_dev_bkey_exists(c, ob->dev);
        offset  = ca->mi.bucket_size - ob->sectors_free;
 
        return ob->ec->new_stripe.data[ob->ec_idx] + (offset << 9);
@@ -1318,7 +1318,7 @@ static int new_stripe_alloc_buckets(struct bch_fs *c, struct ec_stripe_head *h,
                        BUG_ON(j >= h->s->nr_data + h->s->nr_parity);
 
                        h->s->blocks[j] = buckets.v[i];
-                       h->s->new_stripe.key.v.ptrs[j] = ob->ptr;
+                       h->s->new_stripe.key.v.ptrs[j] = bch2_ob_ptr(c, ob);
                        __set_bit(j, h->s->blocks_gotten);
                }
 
@@ -1346,7 +1346,7 @@ static int new_stripe_alloc_buckets(struct bch_fs *c, struct ec_stripe_head *h,
                        BUG_ON(j >= h->s->nr_data);
 
                        h->s->blocks[j] = buckets.v[i];
-                       h->s->new_stripe.key.v.ptrs[j] = ob->ptr;
+                       h->s->new_stripe.key.v.ptrs[j] = bch2_ob_ptr(c, ob);
                        __set_bit(j, h->s->blocks_gotten);
                }
 
@@ -1535,7 +1535,7 @@ void bch2_ec_stop_dev(struct bch_fs *c, struct bch_dev *ca)
                                continue;
 
                        ob = c->open_buckets + h->s->blocks[i];
-                       if (ob->ptr.dev == ca->dev_idx)
+                       if (ob->dev == ca->dev_idx)
                                goto found;
                }
                goto unlock;

diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c
index 7c5ce5b474930bd4023c0650f02fcb496f7b56ef..4fadb41c4c1efc6fc8064e4ae3179a1d01c4cd57 100644 (file)
--- a/fs/bcachefs/journal.c
+++ b/fs/bcachefs/journal.c
@@ -789,7 +789,7 @@ static int __bch2_set_nr_journal_buckets(struct bch_dev *ca, unsigned nr,
                                goto err;
                        }
 
-                       b = sector_to_bucket(ca, ob->ptr.offset);
+                       b = ob->bucket;
                }
 
                if (c)

diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c
index 1d1e2c6fc2e2963f036be1eb4495c1559e07d21b..07e9b214bcb527b783c65167d44a439533a3393c 100644 (file)
--- a/fs/bcachefs/sysfs.c
+++ b/fs/bcachefs/sysfs.c
@@ -10,6 +10,7 @@
 
 #include "bcachefs.h"
 #include "alloc_background.h"
+#include "alloc_foreground.h"
 #include "sysfs.h"
 #include "btree_cache.h"
 #include "btree_io.h"
@@ -723,7 +724,7 @@ static void dev_alloc_debug_to_text(struct printbuf *out, struct bch_dev *ca)
        memset(nr, 0, sizeof(nr));
 
        for (i = 0; i < ARRAY_SIZE(c->open_buckets); i++)
-               nr[c->open_buckets[i].type]++;
+               nr[c->open_buckets[i].data_type]++;
 
        pr_buf(out,
               "\t\t buckets\t sectors      fragmented\n"