mm: Allow GFP_ACCOUNT to be used in alloc_pages_nolock().

Change alloc_pages_nolock() to default to __GFP_COMP when allocating
pages, since upcoming reentrant alloc_slab_page() needs __GFP_COMP.
Also allow __GFP_ACCOUNT flag to be specified,
since most of BPF infra needs __GFP_ACCOUNT except BPF streams.

Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Reviewed-by: Shakeel Butt <shakeel.butt@linux.dev>
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 5ebf26fcdcfa3c813458503f0da988e00f1153fd..0ceb4e09306c4a7098d5a61645396e3b82a1ca30 100644 (file)
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -354,7 +354,7 @@ static inline struct page *alloc_page_vma_noprof(gfp_t gfp,
 }
 #define alloc_page_vma(...)                    alloc_hooks(alloc_page_vma_noprof(__VA_ARGS__))
 
-struct page *alloc_pages_nolock_noprof(int nid, unsigned int order);
+struct page *alloc_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);
 #define alloc_pages_nolock(...)                        alloc_hooks(alloc_pages_nolock_noprof(__VA_ARGS__))
 
 extern unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order);

diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c
index ab592db4a4bf6140a9c82bb14abf3bafbe9689c9..eb6c5a21c2efee96c41f4c5e43d54062694a4859 100644 (file)
--- a/kernel/bpf/stream.c
+++ b/kernel/bpf/stream.c
@@ -83,7 +83,7 @@ static struct bpf_stream_page *bpf_stream_page_replace(void)
        struct bpf_stream_page *stream_page, *old_stream_page;
        struct page *page;
 
-       page = alloc_pages_nolock(NUMA_NO_NODE, 0);
+       page = alloc_pages_nolock(/* Don't account */ 0, NUMA_NO_NODE, 0);
        if (!page)
                return NULL;
        stream_page = page_address(page);

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 0fbfa8532c392ca994e09e3d15b4ec1f747fa827..dbf86f8014de27b147137d8c9bc78fabcdfcf069 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -581,7 +581,7 @@ static bool can_alloc_pages(void)
 static struct page *__bpf_alloc_page(int nid)
 {
        if (!can_alloc_pages())
-               return alloc_pages_nolock(nid, 0);
+               return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0);
 
        return alloc_pages_node(nid,
                                GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index d1d037f97c5fc76f8a7739e8515d7593e0ad44f9..30ccff0283fd3e164cb07c88b3adeafdfe42b289 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -7480,6 +7480,7 @@ static bool __free_unaccepted(struct page *page)
 
 /**
  * alloc_pages_nolock - opportunistic reentrant allocation from any context
+ * @gfp_flags: GFP flags. Only __GFP_ACCOUNT allowed.
  * @nid: node to allocate from
  * @order: allocation order size
  *
@@ -7493,7 +7494,7 @@ static bool __free_unaccepted(struct page *page)
  * Return: allocated page or NULL on failure. NULL does not mean EBUSY or EAGAIN.
  * It means ENOMEM. There is no reason to call it again and expect !NULL.
  */
-struct page *alloc_pages_nolock_noprof(int nid, unsigned int order)
+struct page *alloc_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order)
 {
        /*
         * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
@@ -7515,12 +7516,13 @@ struct page *alloc_pages_nolock_noprof(int nid, unsigned int order)
         * specify it here to highlight that alloc_pages_nolock()
         * doesn't want to deplete reserves.
         */
-       gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC
-                       | __GFP_ACCOUNT;
+       gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP
+                       | gfp_flags;
        unsigned int alloc_flags = ALLOC_TRYLOCK;
        struct alloc_context ac = { };
        struct page *page;
 
+       VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT);
        /*
         * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
         * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
@@ -7558,7 +7560,7 @@ struct page *alloc_pages_nolock_noprof(int nid, unsigned int order)
        if (page)
                set_page_refcounted(page);
 
-       if (memcg_kmem_online() && page &&
+       if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) &&
            unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) {
                free_pages_nolock(page, order);
                page = NULL;