btrfs: index buffer_tree using node size

So far we've been deriving the buffer tree index using the sector size.
But each extent buffer covers multiple sectors. This makes the buffer
tree rather sparse.

For example the typical and quite common configuration uses sector size
of 4KiB and node size of 16KiB. In this case it means the buffer tree is
using up to the maximum of 25% of it's slots. Or in other words at least
75% of the tree slots are wasted as never used.

We can score significant memory savings on the required tree nodes by
indexing the tree using the node size instead. As a result far less
slots are wasted and the tree can now use up to all 100% of it's slots
this way.

Note: This works even with unaligned tree blocks as we can still get
      unique index by doing eb->start >> nodesize_shift.

Getting some stats from running fio write test, there is a bit of
variance.  The values presented in the table below are medians from 5
test runs.  The numbers are:

  - # of allocated ebs in the tree
  - # of leaf tree nodes
  - highest index in the tree (radix tree width)):

ebs / leaves / Index |   bare for-next    |      with fix
---------------------+--------------------+-------------------
	post mount   |   16 /  11 / 10e5c |   16 /  10 / 4240
	post test    | 5810 / 891 / 11cfc | 4420 / 252 / 473a
	post rm	     |  574 / 300 / 10ef0 |  540 / 163 / 46e9

In this case (10GiB filesystem) the height of the tree is still 3 levels
but the 4x width reduction is clearly visible as expected. But since the
tree is more dense we can see the 54-72% reduction of leaf nodes. That's
very close to ideal with this test. It means the tree is getting really
dense with this kind of workload.

Also, the fio results show no performance change.

Reviewed-by: Qu Wenruo <wqu@suse.com>
Signed-off-by: Daniel Vacek <neelx@suse.com>
Reviewed-by: David Sterba <dsterba@suse.com>
Signed-off-by: David Sterba <dsterba@suse.com>

diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 6bc7f7ac381cefbd824dbbad61bd0bddb17787eb..44e7ae4a2e0b6505b4f0485f4435ef054d2420f5 100644 (file)
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -3397,6 +3397,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
        fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids));
 
        fs_info->nodesize = nodesize;
+       fs_info->nodesize_bits = ilog2(nodesize);
        fs_info->sectorsize = sectorsize;
        fs_info->sectorsize_bits = ilog2(sectorsize);
        fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size;

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 594ba0a3b6b9803fef5d0087c8f9ea44118cdcda..cc7f04d56a1039feb28bb9d4fa4b369c0f25ea05 100644 (file)
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -1803,7 +1803,7 @@ static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *e
         */
        spin_lock(&eb->refs_lock);
        if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
-               XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->sectorsize_bits);
+               XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
                unsigned long flags;
 
                set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
@@ -1903,7 +1903,7 @@ static void set_btree_ioerr(struct extent_buffer *eb)
 static void buffer_tree_set_mark(const struct extent_buffer *eb, xa_mark_t mark)
 {
        struct btrfs_fs_info *fs_info = eb->fs_info;
-       XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->sectorsize_bits);
+       XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
        unsigned long flags;
 
        xas_lock_irqsave(&xas, flags);
@@ -1915,7 +1915,7 @@ static void buffer_tree_set_mark(const struct extent_buffer *eb, xa_mark_t mark)
 static void buffer_tree_clear_mark(const struct extent_buffer *eb, xa_mark_t mark)
 {
        struct btrfs_fs_info *fs_info = eb->fs_info;
-       XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->sectorsize_bits);
+       XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
        unsigned long flags;
 
        xas_lock_irqsave(&xas, flags);
@@ -2015,7 +2015,7 @@ static unsigned int buffer_tree_get_ebs_tag(struct btrfs_fs_info *fs_info,
        rcu_read_lock();
        while ((eb = find_get_eb(&xas, end, tag)) != NULL) {
                if (!eb_batch_add(batch, eb)) {
-                       *start = ((eb->start + eb->len) >> fs_info->sectorsize_bits);
+                       *start = ((eb->start + eb->len) >> fs_info->nodesize_bits);
                        goto out;
                }
        }
@@ -2037,7 +2037,7 @@ static struct extent_buffer *find_extent_buffer_nolock(
                struct btrfs_fs_info *fs_info, u64 start)
 {
        struct extent_buffer *eb;
-       unsigned long index = (start >> fs_info->sectorsize_bits);
+       unsigned long index = (start >> fs_info->nodesize_bits);
 
        rcu_read_lock();
        eb = xa_load(&fs_info->buffer_tree, index);
@@ -2143,8 +2143,8 @@ void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start,
                                      u64 end)
 {
        struct eb_batch batch;
-       unsigned long start_index = (start >> fs_info->sectorsize_bits);
-       unsigned long end_index = (end >> fs_info->sectorsize_bits);
+       unsigned long start_index = (start >> fs_info->nodesize_bits);
+       unsigned long end_index = (end >> fs_info->nodesize_bits);
 
        eb_batch_init(&batch);
        while (start_index <= end_index) {
@@ -2180,7 +2180,7 @@ int btree_write_cache_pages(struct address_space *mapping,
 
        eb_batch_init(&batch);
        if (wbc->range_cyclic) {
-               index = ((mapping->writeback_index << PAGE_SHIFT) >> fs_info->sectorsize_bits);
+               index = ((mapping->writeback_index << PAGE_SHIFT) >> fs_info->nodesize_bits);
                end = -1;
 
                /*
@@ -2189,8 +2189,8 @@ int btree_write_cache_pages(struct address_space *mapping,
                 */
                scanned = (index == 0);
        } else {
-               index = (wbc->range_start >> fs_info->sectorsize_bits);
-               end = (wbc->range_end >> fs_info->sectorsize_bits);
+               index = (wbc->range_start >> fs_info->nodesize_bits);
+               end = (wbc->range_end >> fs_info->nodesize_bits);
 
                scanned = 1;
        }
@@ -3070,7 +3070,7 @@ struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
        eb->fs_info = fs_info;
 again:
        xa_lock_irq(&fs_info->buffer_tree);
-       exists = __xa_cmpxchg(&fs_info->buffer_tree, start >> fs_info->sectorsize_bits,
+       exists = __xa_cmpxchg(&fs_info->buffer_tree, start >> fs_info->nodesize_bits,
                              NULL, eb, GFP_NOFS);
        if (xa_is_err(exists)) {
                ret = xa_err(exists);
@@ -3386,7 +3386,7 @@ reallocate:
 again:
        xa_lock_irq(&fs_info->buffer_tree);
        existing_eb = __xa_cmpxchg(&fs_info->buffer_tree,
-                                  start >> fs_info->sectorsize_bits, NULL, eb,
+                                  start >> fs_info->nodesize_bits, NULL, eb,
                                   GFP_NOFS);
        if (xa_is_err(existing_eb)) {
                ret = xa_err(existing_eb);
@@ -3489,7 +3489,7 @@ static int release_extent_buffer(struct extent_buffer *eb)
                 * in this case.
                 */
                xa_cmpxchg_irq(&fs_info->buffer_tree,
-                              eb->start >> fs_info->sectorsize_bits, eb, NULL,
+                              eb->start >> fs_info->nodesize_bits, eb, NULL,
                               GFP_ATOMIC);
 
                btrfs_leak_debug_del_eb(eb);
@@ -4331,9 +4331,9 @@ static int try_release_subpage_extent_buffer(struct folio *folio)
 {
        struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
        struct extent_buffer *eb;
-       unsigned long start = (folio_pos(folio) >> fs_info->sectorsize_bits);
+       unsigned long start = (folio_pos(folio) >> fs_info->nodesize_bits);
        unsigned long index = start;
-       unsigned long end = index + (PAGE_SIZE >> fs_info->sectorsize_bits) - 1;
+       unsigned long end = index + (PAGE_SIZE >> fs_info->nodesize_bits) - 1;
        int ret;
 
        xa_lock_irq(&fs_info->buffer_tree);

diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h
index 5154ad390f319ea9c70de024f8af03c4b5ad402f..8cc07cc70b128357b7950c83b5e438a1f431e500 100644 (file)
--- a/fs/btrfs/fs.h
+++ b/fs/btrfs/fs.h
@@ -773,7 +773,7 @@ struct btrfs_fs_info {
 
        struct btrfs_delayed_root *delayed_root;
 
-       /* Entries are eb->start / sectorsize */
+       /* Entries are eb->start >> nodesize_bits */
        struct xarray buffer_tree;
 
        /* Next backup root to be overwritten */
@@ -805,6 +805,7 @@ struct btrfs_fs_info {
 
        /* Cached block sizes */
        u32 nodesize;
+       u32 nodesize_bits;
        u32 sectorsize;
        /* ilog2 of sectorsize, use to avoid 64bit division */
        u32 sectorsize_bits;