From: Greg Kroah-Hartman Date: Wed, 2 Oct 2024 09:58:25 +0000 (+0200) Subject: 6.1-stable patches X-Git-Tag: v6.6.54~39 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=e5af707384596058eceef1349db9d9b518864cb3;p=thirdparty%2Fkernel%2Fstable-queue.git 6.1-stable patches added patches: lib-xarray-introduce-a-new-helper-xas_get_order.patch mm-filemap-optimize-filemap-folio-adding.patch mm-filemap-return-early-if-failed-to-allocate-memory-for-split.patch --- diff --git a/queue-6.1/lib-xarray-introduce-a-new-helper-xas_get_order.patch b/queue-6.1/lib-xarray-introduce-a-new-helper-xas_get_order.patch new file mode 100644 index 00000000000..d4d54c0b7ca --- /dev/null +++ b/queue-6.1/lib-xarray-introduce-a-new-helper-xas_get_order.patch @@ -0,0 +1,180 @@ +From stable+bounces-78585-greg=kroah.com@vger.kernel.org Tue Oct 1 23:09:46 2024 +From: Kairui Song +Date: Wed, 2 Oct 2024 05:06:24 +0800 +Subject: lib/xarray: introduce a new helper xas_get_order +To: stable@vger.kernel.org, Greg KH +Cc: Matthew Wilcox , Jens Axboe , David Howells , Dave Chinner , Christian Theune , Christian Brauner , Chris Mason , Sam James , Daniel Dao , Linus Torvalds , Kairui Song , Andrew Morton +Message-ID: <20241001210625.95825-3-ryncsn@gmail.com> + +From: Kairui Song + +commit a4864671ca0bf51c8e78242951741df52c06766f upstream. + +It can be used after xas_load to check the order of loaded entries. +Compared to xa_get_order, it saves an XA_STATE and avoid a rewalk. + +Added new test for xas_get_order, to make the test work, we have to export +xas_get_order with EXPORT_SYMBOL_GPL. + +Also fix a sparse warning by checking the slot value with xa_entry instead +of accessing it directly, as suggested by Matthew Wilcox. + +[kasong@tencent.com: simplify comment, sparse warning fix, per Matthew Wilcox] + Link: https://lkml.kernel.org/r/20240416071722.45997-4-ryncsn@gmail.com +Link: https://lkml.kernel.org/r/20240415171857.19244-4-ryncsn@gmail.com +Signed-off-by: Kairui Song +Cc: Matthew Wilcox (Oracle) +Signed-off-by: Andrew Morton +Stable-dep-of: 6758c1128ceb ("mm/filemap: optimize filemap folio adding") +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/xarray.h | 6 ++++++ + lib/test_xarray.c | 34 ++++++++++++++++++++++++++++++++++ + lib/xarray.c | 49 +++++++++++++++++++++++++++++++------------------ + 3 files changed, 71 insertions(+), 18 deletions(-) + +--- a/include/linux/xarray.h ++++ b/include/linux/xarray.h +@@ -1530,6 +1530,7 @@ void xas_create_range(struct xa_state *) + + #ifdef CONFIG_XARRAY_MULTI + int xa_get_order(struct xarray *, unsigned long index); ++int xas_get_order(struct xa_state *xas); + void xas_split(struct xa_state *, void *entry, unsigned int order); + void xas_split_alloc(struct xa_state *, void *entry, unsigned int order, gfp_t); + #else +@@ -1537,6 +1538,11 @@ static inline int xa_get_order(struct xa + { + return 0; + } ++ ++static inline int xas_get_order(struct xa_state *xas) ++{ ++ return 0; ++} + + static inline void xas_split(struct xa_state *xas, void *entry, + unsigned int order) +--- a/lib/test_xarray.c ++++ b/lib/test_xarray.c +@@ -1756,6 +1756,39 @@ static noinline void check_get_order(str + } + } + ++static noinline void check_xas_get_order(struct xarray *xa) ++{ ++ XA_STATE(xas, xa, 0); ++ ++ unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 20 : 1; ++ unsigned int order; ++ unsigned long i, j; ++ ++ for (order = 0; order < max_order; order++) { ++ for (i = 0; i < 10; i++) { ++ xas_set_order(&xas, i << order, order); ++ do { ++ xas_lock(&xas); ++ xas_store(&xas, xa_mk_value(i)); ++ xas_unlock(&xas); ++ } while (xas_nomem(&xas, GFP_KERNEL)); ++ ++ for (j = i << order; j < (i + 1) << order; j++) { ++ xas_set_order(&xas, j, 0); ++ rcu_read_lock(); ++ xas_load(&xas); ++ XA_BUG_ON(xa, xas_get_order(&xas) != order); ++ rcu_read_unlock(); ++ } ++ ++ xas_lock(&xas); ++ xas_set_order(&xas, i << order, order); ++ xas_store(&xas, NULL); ++ xas_unlock(&xas); ++ } ++ } ++} ++ + static noinline void check_destroy(struct xarray *xa) + { + unsigned long index; +@@ -1805,6 +1838,7 @@ static int xarray_checks(void) + check_reserve(&xa0); + check_multi_store(&array); + check_get_order(&array); ++ check_xas_get_order(&array); + check_xa_alloc(); + check_find(&array); + check_find_entry(&array); +--- a/lib/xarray.c ++++ b/lib/xarray.c +@@ -1752,39 +1752,52 @@ unlock: + EXPORT_SYMBOL(xa_store_range); + + /** +- * xa_get_order() - Get the order of an entry. +- * @xa: XArray. +- * @index: Index of the entry. ++ * xas_get_order() - Get the order of an entry. ++ * @xas: XArray operation state. ++ * ++ * Called after xas_load, the xas should not be in an error state. + * + * Return: A number between 0 and 63 indicating the order of the entry. + */ +-int xa_get_order(struct xarray *xa, unsigned long index) ++int xas_get_order(struct xa_state *xas) + { +- XA_STATE(xas, xa, index); +- void *entry; + int order = 0; + +- rcu_read_lock(); +- entry = xas_load(&xas); +- +- if (!entry) +- goto unlock; +- +- if (!xas.xa_node) +- goto unlock; ++ if (!xas->xa_node) ++ return 0; + + for (;;) { +- unsigned int slot = xas.xa_offset + (1 << order); ++ unsigned int slot = xas->xa_offset + (1 << order); + + if (slot >= XA_CHUNK_SIZE) + break; +- if (!xa_is_sibling(xas.xa_node->slots[slot])) ++ if (!xa_is_sibling(xa_entry(xas->xa, xas->xa_node, slot))) + break; + order++; + } + +- order += xas.xa_node->shift; +-unlock: ++ order += xas->xa_node->shift; ++ return order; ++} ++EXPORT_SYMBOL_GPL(xas_get_order); ++ ++/** ++ * xa_get_order() - Get the order of an entry. ++ * @xa: XArray. ++ * @index: Index of the entry. ++ * ++ * Return: A number between 0 and 63 indicating the order of the entry. ++ */ ++int xa_get_order(struct xarray *xa, unsigned long index) ++{ ++ XA_STATE(xas, xa, index); ++ int order = 0; ++ void *entry; ++ ++ rcu_read_lock(); ++ entry = xas_load(&xas); ++ if (entry) ++ order = xas_get_order(&xas); + rcu_read_unlock(); + + return order; diff --git a/queue-6.1/mm-filemap-optimize-filemap-folio-adding.patch b/queue-6.1/mm-filemap-optimize-filemap-folio-adding.patch new file mode 100644 index 00000000000..93bf3ab73bb --- /dev/null +++ b/queue-6.1/mm-filemap-optimize-filemap-folio-adding.patch @@ -0,0 +1,250 @@ +From stable+bounces-78586-greg=kroah.com@vger.kernel.org Tue Oct 1 23:09:46 2024 +From: Kairui Song +Date: Wed, 2 Oct 2024 05:06:25 +0800 +Subject: mm/filemap: optimize filemap folio adding +To: stable@vger.kernel.org, Greg KH +Cc: Matthew Wilcox , Jens Axboe , David Howells , Dave Chinner , Christian Theune , Christian Brauner , Chris Mason , Sam James , Daniel Dao , Linus Torvalds , Kairui Song , Andrew Morton +Message-ID: <20241001210625.95825-4-ryncsn@gmail.com> + +From: Kairui Song + +commit 6758c1128ceb45d1a35298912b974eb4895b7dd9 upstream. + +Instead of doing multiple tree walks, do one optimism range check with +lock hold, and exit if raced with another insertion. If a shadow exists, +check it with a new xas_get_order helper before releasing the lock to +avoid redundant tree walks for getting its order. + +Drop the lock and do the allocation only if a split is needed. + +In the best case, it only need to walk the tree once. If it needs to +alloc and split, 3 walks are issued (One for first ranged conflict check +and order retrieving, one for the second check after allocation, one for +the insert after split). + +Testing with 4K pages, in an 8G cgroup, with 16G brd as block device: + + echo 3 > /proc/sys/vm/drop_caches + + fio -name=cached --numjobs=16 --filename=/mnt/test.img \ + --buffered=1 --ioengine=mmap --rw=randread --time_based \ + --ramp_time=30s --runtime=5m --group_reporting + +Before: +bw ( MiB/s): min= 1027, max= 3520, per=100.00%, avg=2445.02, stdev=18.90, samples=8691 +iops : min=263001, max=901288, avg=625924.36, stdev=4837.28, samples=8691 + +After (+7.3%): +bw ( MiB/s): min= 493, max= 3947, per=100.00%, avg=2625.56, stdev=25.74, samples=8651 +iops : min=126454, max=1010681, avg=672142.61, stdev=6590.48, samples=8651 + +Test result with THP (do a THP randread then switch to 4K page in hope it +issues a lot of splitting): + + echo 3 > /proc/sys/vm/drop_caches + + fio -name=cached --numjobs=16 --filename=/mnt/test.img \ + --buffered=1 --ioengine=mmap -thp=1 --readonly \ + --rw=randread --time_based --ramp_time=30s --runtime=10m \ + --group_reporting + + fio -name=cached --numjobs=16 --filename=/mnt/test.img \ + --buffered=1 --ioengine=mmap \ + --rw=randread --time_based --runtime=5s --group_reporting + +Before: +bw ( KiB/s): min= 4141, max=14202, per=100.00%, avg=7935.51, stdev=96.85, samples=18976 +iops : min= 1029, max= 3548, avg=1979.52, stdev=24.23, samples=18976· + +READ: bw=4545B/s (4545B/s), 4545B/s-4545B/s (4545B/s-4545B/s), io=64.0KiB (65.5kB), run=14419-14419msec + +After (+12.5%): +bw ( KiB/s): min= 4611, max=15370, per=100.00%, avg=8928.74, stdev=105.17, samples=19146 +iops : min= 1151, max= 3842, avg=2231.27, stdev=26.29, samples=19146 + +READ: bw=4635B/s (4635B/s), 4635B/s-4635B/s (4635B/s-4635B/s), io=64.0KiB (65.5kB), run=14137-14137msec + +The performance is better for both 4K (+7.5%) and THP (+12.5%) cached read. + +Link: https://lkml.kernel.org/r/20240415171857.19244-5-ryncsn@gmail.com +Signed-off-by: Kairui Song +Cc: Matthew Wilcox (Oracle) +Signed-off-by: Andrew Morton +Closes: https://lore.kernel.org/linux-mm/A5A976CB-DB57-4513-A700-656580488AB6@flyingcircus.io/ +[ kasong@tencent.com: minor adjustment of variable declarations ] +Signed-off-by: Greg Kroah-Hartman +--- + lib/test_xarray.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ + mm/filemap.c | 53 +++++++++++++++++++++++++++++++++++------------- + 2 files changed, 98 insertions(+), 14 deletions(-) + +--- a/lib/test_xarray.c ++++ b/lib/test_xarray.c +@@ -1789,6 +1789,64 @@ static noinline void check_xas_get_order + } + } + ++static noinline void check_xas_conflict_get_order(struct xarray *xa) ++{ ++ XA_STATE(xas, xa, 0); ++ ++ void *entry; ++ int only_once; ++ unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 20 : 1; ++ unsigned int order; ++ unsigned long i, j, k; ++ ++ for (order = 0; order < max_order; order++) { ++ for (i = 0; i < 10; i++) { ++ xas_set_order(&xas, i << order, order); ++ do { ++ xas_lock(&xas); ++ xas_store(&xas, xa_mk_value(i)); ++ xas_unlock(&xas); ++ } while (xas_nomem(&xas, GFP_KERNEL)); ++ ++ /* ++ * Ensure xas_get_order works with xas_for_each_conflict. ++ */ ++ j = i << order; ++ for (k = 0; k < order; k++) { ++ only_once = 0; ++ xas_set_order(&xas, j + (1 << k), k); ++ xas_lock(&xas); ++ xas_for_each_conflict(&xas, entry) { ++ XA_BUG_ON(xa, entry != xa_mk_value(i)); ++ XA_BUG_ON(xa, xas_get_order(&xas) != order); ++ only_once++; ++ } ++ XA_BUG_ON(xa, only_once != 1); ++ xas_unlock(&xas); ++ } ++ ++ if (order < max_order - 1) { ++ only_once = 0; ++ xas_set_order(&xas, (i & ~1UL) << order, order + 1); ++ xas_lock(&xas); ++ xas_for_each_conflict(&xas, entry) { ++ XA_BUG_ON(xa, entry != xa_mk_value(i)); ++ XA_BUG_ON(xa, xas_get_order(&xas) != order); ++ only_once++; ++ } ++ XA_BUG_ON(xa, only_once != 1); ++ xas_unlock(&xas); ++ } ++ ++ xas_set_order(&xas, i << order, order); ++ xas_lock(&xas); ++ xas_store(&xas, NULL); ++ xas_unlock(&xas); ++ } ++ } ++} ++ ++ + static noinline void check_destroy(struct xarray *xa) + { + unsigned long index; +@@ -1839,6 +1897,7 @@ static int xarray_checks(void) + check_multi_store(&array); + check_get_order(&array); + check_xas_get_order(&array); ++ check_xas_conflict_get_order(&array); + check_xa_alloc(); + check_find(&array); + check_find_entry(&array); +--- a/mm/filemap.c ++++ b/mm/filemap.c +@@ -841,6 +841,8 @@ noinline int __filemap_add_folio(struct + { + XA_STATE(xas, &mapping->i_pages, index); + int huge = folio_test_hugetlb(folio); ++ void *alloced_shadow = NULL; ++ int alloced_order = 0; + bool charged = false; + long nr = 1; + +@@ -863,16 +865,10 @@ noinline int __filemap_add_folio(struct + folio->mapping = mapping; + folio->index = xas.xa_index; + +- do { +- unsigned int order = xa_get_order(xas.xa, xas.xa_index); ++ for (;;) { ++ int order = -1, split_order = 0; + void *entry, *old = NULL; + +- if (order > folio_order(folio)) { +- xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index), +- order, gfp); +- if (xas_error(&xas)) +- goto error; +- } + xas_lock_irq(&xas); + xas_for_each_conflict(&xas, entry) { + old = entry; +@@ -880,19 +876,33 @@ noinline int __filemap_add_folio(struct + xas_set_err(&xas, -EEXIST); + goto unlock; + } ++ /* ++ * If a larger entry exists, ++ * it will be the first and only entry iterated. ++ */ ++ if (order == -1) ++ order = xas_get_order(&xas); ++ } ++ ++ /* entry may have changed before we re-acquire the lock */ ++ if (alloced_order && (old != alloced_shadow || order != alloced_order)) { ++ xas_destroy(&xas); ++ alloced_order = 0; + } + + if (old) { +- if (shadowp) +- *shadowp = old; +- /* entry may have been split before we acquired lock */ +- order = xa_get_order(xas.xa, xas.xa_index); +- if (order > folio_order(folio)) { ++ if (order > 0 && order > folio_order(folio)) { + /* How to handle large swap entries? */ + BUG_ON(shmem_mapping(mapping)); ++ if (!alloced_order) { ++ split_order = order; ++ goto unlock; ++ } + xas_split(&xas, old, order); + xas_reset(&xas); + } ++ if (shadowp) ++ *shadowp = old; + } + + xas_store(&xas, folio); +@@ -908,9 +918,24 @@ noinline int __filemap_add_folio(struct + __lruvec_stat_mod_folio(folio, + NR_FILE_THPS, nr); + } ++ + unlock: + xas_unlock_irq(&xas); +- } while (xas_nomem(&xas, gfp)); ++ ++ /* split needed, alloc here and retry. */ ++ if (split_order) { ++ xas_split_alloc(&xas, old, split_order, gfp); ++ if (xas_error(&xas)) ++ goto error; ++ alloced_shadow = old; ++ alloced_order = split_order; ++ xas_reset(&xas); ++ continue; ++ } ++ ++ if (!xas_nomem(&xas, gfp)) ++ break; ++ } + + if (xas_error(&xas)) + goto error; diff --git a/queue-6.1/mm-filemap-return-early-if-failed-to-allocate-memory-for-split.patch b/queue-6.1/mm-filemap-return-early-if-failed-to-allocate-memory-for-split.patch new file mode 100644 index 00000000000..3f0bfebd485 --- /dev/null +++ b/queue-6.1/mm-filemap-return-early-if-failed-to-allocate-memory-for-split.patch @@ -0,0 +1,43 @@ +From stable+bounces-78584-greg=kroah.com@vger.kernel.org Tue Oct 1 23:09:40 2024 +From: Kairui Song +Date: Wed, 2 Oct 2024 05:06:23 +0800 +Subject: mm/filemap: return early if failed to allocate memory for split +To: stable@vger.kernel.org, Greg KH +Cc: Matthew Wilcox , Jens Axboe , David Howells , Dave Chinner , Christian Theune , Christian Brauner , Chris Mason , Sam James , Daniel Dao , Linus Torvalds , Kairui Song , Andrew Morton +Message-ID: <20241001210625.95825-2-ryncsn@gmail.com> + +From: Kairui Song + +commit de60fd8ddeda2b41fbe11df11733838c5f684616 upstream. + +xas_split_alloc could fail with NOMEM, and in such case, it should abort +early instead of keep going and fail the xas_split below. + +Link: https://lkml.kernel.org/r/20240416071722.45997-1-ryncsn@gmail.com +Link: https://lkml.kernel.org/r/20240415171857.19244-1-ryncsn@gmail.com +Link: https://lkml.kernel.org/r/20240415171857.19244-2-ryncsn@gmail.com +Signed-off-by: Kairui Song +Acked-by: Matthew Wilcox (Oracle) +Signed-off-by: Andrew Morton +Stable-dep-of: 6758c1128ceb ("mm/filemap: optimize filemap folio adding") +Signed-off-by: Greg Kroah-Hartman +--- + mm/filemap.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +--- a/mm/filemap.c ++++ b/mm/filemap.c +@@ -867,9 +867,12 @@ noinline int __filemap_add_folio(struct + unsigned int order = xa_get_order(xas.xa, xas.xa_index); + void *entry, *old = NULL; + +- if (order > folio_order(folio)) ++ if (order > folio_order(folio)) { + xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index), + order, gfp); ++ if (xas_error(&xas)) ++ goto error; ++ } + xas_lock_irq(&xas); + xas_for_each_conflict(&xas, entry) { + old = entry; diff --git a/queue-6.1/series b/queue-6.1/series index f6402b87201..04e2e8f73b0 100644 --- a/queue-6.1/series +++ b/queue-6.1/series @@ -368,3 +368,6 @@ x86-idtentry-incorporate-definitions-declarations-of.patch x86-entry-remove-unwanted-instrumentation-in-common_.patch cpuidle-adjust-includes-to-remove-of_device.h.patch cpuidle-riscv-sbi-use-scoped-device-node-handling-to.patch +mm-filemap-return-early-if-failed-to-allocate-memory-for-split.patch +lib-xarray-introduce-a-new-helper-xas_get_order.patch +mm-filemap-optimize-filemap-folio-adding.patch