bcachefs: New in-memory array for bucket gens

The main in-memory bucket array is going away, but we'll still need to
keep bucket generations in memory, at least for now - ptr_stale() needs
to be an efficient operation.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>

diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c
index 8831b2a0303a85fc9156a321fa874494b80d978d..45becfb1ffe9050e6b67456171d9e290d9f4b8e5 100644 (file)
--- a/fs/bcachefs/alloc_background.c
+++ b/fs/bcachefs/alloc_background.c
@@ -354,6 +354,7 @@ static int bch2_alloc_read_fn(struct btree_trans *trans, struct bkey_s_c k)
        g = bucket(ca, k.k->p.offset);
        u = bch2_alloc_unpack(k);
 
+       *bucket_gen(ca, k.k->p.offset) = u.gen;
        g->_mark.gen            = u.gen;
        g->_mark.data_type      = u.data_type;
        g->_mark.dirty_sectors  = u.dirty_sectors;
@@ -748,6 +749,7 @@ static int bch2_invalidate_one_bucket(struct bch_fs *c, struct bch_dev *ca,
            !bucket_needs_journal_commit(m, c->journal.last_seq_ondisk)) {
                BUG_ON(m.data_type);
                bucket_cmpxchg(g, m, m.gen++);
+               *bucket_gen(ca, b) = m.gen;
                percpu_up_read(&c->mark_lock);
                goto out;
        }

diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c
index 7506d54c854b9dd0f90ff9b4b94c840b5e8a08bd..e2038032b872ade890296cbb343be0161dd0f3bb 100644 (file)
--- a/fs/bcachefs/alloc_foreground.c
+++ b/fs/bcachefs/alloc_foreground.c
@@ -161,7 +161,7 @@ static void verify_not_stale(struct bch_fs *c, const struct open_buckets *obs)
        open_bucket_for_each(c, obs, ob, i) {
                struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
 
-               BUG_ON(bucket(ca, ob->bucket)->mark.gen != ob->gen);
+               BUG_ON(*bucket_gen(ca, ob->bucket) != ob->gen);
        }
        rcu_read_unlock();
 #endif
@@ -273,7 +273,7 @@ out:
        ob->sectors_free = ca->mi.bucket_size;
        ob->alloc_reserve = reserve;
        ob->dev         = ca->dev_idx;
-       ob->gen         = bucket(ca, b)->mark.gen;
+       ob->gen         = *bucket_gen(ca, b);
        ob->bucket      = b;
        spin_unlock(&ob->lock);
 

diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index 6c686be28b39a6650be468e37956f59a4ed66f84..c282086079fbcc1d2fd3420fd60326bd2426dbb7 100644 (file)
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -445,6 +445,7 @@ struct bch_dev {
         * Or rcu_read_lock(), but only for ptr_stale():
         */
        struct bucket_array __rcu *buckets[2];
+       struct bucket_gens      *bucket_gens;
        unsigned long           *buckets_nouse;
        struct rw_semaphore     bucket_lock;
 

diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index c1b0d0be07a6f9e78cfbdce51c46429099529508..4b7fe4a5def93793fa88f5ea01f94fb7a07a478b 100644 (file)
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -535,6 +535,20 @@ void bch2_mark_alloc_bucket(struct bch_fs *c, struct bch_dev *ca,
        BUG_ON(owned_by_allocator == old.owned_by_allocator);
 }
 
+static inline u8 bkey_alloc_gen(struct bkey_s_c k)
+{
+       switch (k.k->type) {
+       case KEY_TYPE_alloc:
+               return bkey_s_c_to_alloc(k).v->gen;
+       case KEY_TYPE_alloc_v2:
+               return bkey_s_c_to_alloc_v2(k).v->gen;
+       case KEY_TYPE_alloc_v3:
+               return bkey_s_c_to_alloc_v3(k).v->gen;
+       default:
+               return 0;
+       }
+}
+
 static int bch2_mark_alloc(struct btree_trans *trans,
                           struct bkey_s_c old, struct bkey_s_c new,
                           unsigned flags)
@@ -573,9 +587,13 @@ static int bch2_mark_alloc(struct btree_trans *trans,
        if (new.k->p.offset >= ca->mi.nbuckets)
                return 0;
 
+       u = bch2_alloc_unpack(new);
+
        percpu_down_read(&c->mark_lock);
+       if (!gc && u.gen != bkey_alloc_gen(old))
+               *bucket_gen(ca, new.k->p.offset) = u.gen;
+
        g = __bucket(ca, new.k->p.offset, gc);
-       u = bch2_alloc_unpack(new);
 
        old_m = bucket_cmpxchg(g, m, ({
                m.gen                   = u.gen;
@@ -2131,9 +2149,18 @@ static void buckets_free_rcu(struct rcu_head *rcu)
                buckets->nbuckets * sizeof(struct bucket));
 }
 
+static void bucket_gens_free_rcu(struct rcu_head *rcu)
+{
+       struct bucket_gens *buckets =
+               container_of(rcu, struct bucket_gens, rcu);
+
+       kvpfree(buckets, sizeof(struct bucket_array) + buckets->nbuckets);
+}
+
 int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
 {
        struct bucket_array *buckets = NULL, *old_buckets = NULL;
+       struct bucket_gens *bucket_gens = NULL, *old_bucket_gens = NULL;
        unsigned long *buckets_nouse = NULL;
        alloc_fifo      free[RESERVE_NR];
        alloc_fifo      free_inc;
@@ -2157,6 +2184,8 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
        if (!(buckets           = kvpmalloc(sizeof(struct bucket_array) +
                                            nbuckets * sizeof(struct bucket),
                                            GFP_KERNEL|__GFP_ZERO)) ||
+           !(bucket_gens       = kvpmalloc(sizeof(struct bucket_gens) + nbuckets,
+                                           GFP_KERNEL|__GFP_ZERO)) ||
            !(buckets_nouse     = kvpmalloc(BITS_TO_LONGS(nbuckets) *
                                            sizeof(unsigned long),
                                            GFP_KERNEL|__GFP_ZERO)) ||
@@ -2169,6 +2198,8 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
 
        buckets->first_bucket   = ca->mi.first_bucket;
        buckets->nbuckets       = nbuckets;
+       bucket_gens->first_bucket = ca->mi.first_bucket;
+       bucket_gens->nbuckets   = nbuckets;
 
        bch2_copygc_stop(c);
 
@@ -2179,6 +2210,7 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
        }
 
        old_buckets = bucket_array(ca);
+       old_bucket_gens = rcu_dereference_protected(ca->bucket_gens, 1);
 
        if (resize) {
                size_t n = min(buckets->nbuckets, old_buckets->nbuckets);
@@ -2186,13 +2218,18 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
                memcpy(buckets->b,
                       old_buckets->b,
                       n * sizeof(struct bucket));
+               memcpy(bucket_gens->b,
+                      old_bucket_gens->b,
+                      n);
                memcpy(buckets_nouse,
                       ca->buckets_nouse,
                       BITS_TO_LONGS(n) * sizeof(unsigned long));
        }
 
        rcu_assign_pointer(ca->buckets[0], buckets);
-       buckets = old_buckets;
+       rcu_assign_pointer(ca->bucket_gens, bucket_gens);
+       buckets         = old_buckets;
+       bucket_gens     = old_bucket_gens;
 
        swap(ca->buckets_nouse, buckets_nouse);
 
@@ -2226,6 +2263,8 @@ err:
                free_fifo(&free[i]);
        kvpfree(buckets_nouse,
                BITS_TO_LONGS(nbuckets) * sizeof(unsigned long));
+       if (bucket_gens)
+               call_rcu(&old_buckets->rcu, bucket_gens_free_rcu);
        if (buckets)
                call_rcu(&old_buckets->rcu, buckets_free_rcu);
 

diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 61baaa66b6b5eff2525b5dfb2fc89f55375b51ce..6eeb95068b3b22d473a73a4c8c68fccc5719f08a 100644 (file)
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -63,6 +63,24 @@ static inline struct bucket *bucket(struct bch_dev *ca, size_t b)
        return __bucket(ca, b, false);
 }
 
+static inline struct bucket_gens *bucket_gens(struct bch_dev *ca)
+{
+       return rcu_dereference_check(ca->bucket_gens,
+                                    !ca->fs ||
+                                    percpu_rwsem_is_held(&ca->fs->mark_lock) ||
+                                    lockdep_is_held(&ca->fs->gc_lock) ||
+                                    lockdep_is_held(&ca->bucket_lock));
+
+}
+
+static inline u8 *bucket_gen(struct bch_dev *ca, size_t b)
+{
+       struct bucket_gens *gens = bucket_gens(ca);
+
+       BUG_ON(b < gens->first_bucket || b >= gens->nbuckets);
+       return gens->b + b;
+}
+
 /*
  * bucket_gc_gen() returns the difference between the bucket's current gen and
  * the oldest gen of any pointer into that bucket in the btree.
@@ -123,7 +141,7 @@ static inline u8 ptr_stale(struct bch_dev *ca,
        u8 ret;
 
        rcu_read_lock();
-       ret = gen_after(PTR_BUCKET(ca, ptr)->mark.gen, ptr->gen);
+       ret = gen_after(*bucket_gen(ca, PTR_BUCKET_NR(ca, ptr)), ptr->gen);
        rcu_read_unlock();
 
        return ret;

diff --git a/fs/bcachefs/buckets_types.h b/fs/bcachefs/buckets_types.h
index b2de2995c5e7f5b157a0f0359a1eac8b6bc5672d..18bca269b7503f2a7d589d6842aee118d31a7e7e 100644 (file)
--- a/fs/bcachefs/buckets_types.h
+++ b/fs/bcachefs/buckets_types.h
@@ -52,6 +52,13 @@ struct bucket_array {
        struct bucket           b[];
 };
 
+struct bucket_gens {
+       struct rcu_head         rcu;
+       u16                     first_bucket;
+       size_t                  nbuckets;
+       u8                      b[];
+};
+
 struct bch_dev_usage {
        u64                     buckets_ec;
        u64                     buckets_unavailable;