Merge tag 'percpu-for-6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis...

Pull percpu updates from Dennis Zhou:
 "Enable percpu page allocator for RISC-V.

  There are RISC-V configurations with sparse NUMA configurations and
  small vmalloc space causing dynamic percpu allocations to fail as the
  backing chunk stride is too far apart"

* tag 'percpu-for-6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu:
  riscv: Enable pcpu page first chunk allocator
  mm: Introduce flush_cache_vmap_early()

diff --git a/arch/arc/include/asm/cacheflush.h b/arch/arc/include/asm/cacheflush.h
index 563af3e75f01f2acb576110ef3210922a8dfb672..329c94cd45d8f68b3fa7866f92a6367a59f23530 100644 (file)
--- a/arch/arc/include/asm/cacheflush.h
+++ b/arch/arc/include/asm/cacheflush.h
@@ -40,6 +40,7 @@ void dma_cache_wback(phys_addr_t start, unsigned long sz);
 
 /* TBD: optimize this */
 #define flush_cache_vmap(start, end)           flush_cache_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_cache_all()
 
 #define flush_cache_dup_mm(mm)                 /* called on fork (VIVT only) */

diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h
index f6181f69577fe538dc4a13a85cc479784ad9ebef..1075534b0a2eeba73be7d0e83b9d0c995e1f8cf9 100644 (file)
--- a/arch/arm/include/asm/cacheflush.h
+++ b/arch/arm/include/asm/cacheflush.h
@@ -340,6 +340,8 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end)
                dsb(ishst);
 }
 
+#define flush_cache_vmap_early(start, end)     do { } while (0)
+
 static inline void flush_cache_vunmap(unsigned long start, unsigned long end)
 {
        if (!cache_is_vipt_nonaliasing())

diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h
index 908d8b0bc4fdc645f6ff5b06965df704a97f49cf..d011a81575d21e08a85621e8a58c59ba8e5e7dd8 100644 (file)
--- a/arch/csky/abiv1/inc/abi/cacheflush.h
+++ b/arch/csky/abiv1/inc/abi/cacheflush.h
@@ -43,6 +43,7 @@ static inline void flush_anon_page(struct vm_area_struct *vma,
  */
 extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 #define flush_cache_vmap(start, end)           cache_wbinv_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         cache_wbinv_all()
 
 #define flush_icache_range(start, end)         cache_wbinv_range(start, end)

diff --git a/arch/csky/abiv2/inc/abi/cacheflush.h b/arch/csky/abiv2/inc/abi/cacheflush.h
index 40be16907267d673581278512742567213187a38..6513ac5d257888fbd41385c9263305dfefd18de6 100644 (file)
--- a/arch/csky/abiv2/inc/abi/cacheflush.h
+++ b/arch/csky/abiv2/inc/abi/cacheflush.h
@@ -41,6 +41,7 @@ void flush_icache_mm_range(struct mm_struct *mm,
 void flush_icache_deferred(struct mm_struct *mm);
 
 #define flush_cache_vmap(start, end)           do { } while (0)
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         do { } while (0)
 
 #define copy_to_user_page(vma, page, vaddr, dst, src, len) \

diff --git a/arch/m68k/include/asm/cacheflush_mm.h b/arch/m68k/include/asm/cacheflush_mm.h
index ed12358c4783b468ae834106925732ef5875c772..9a71b0148461a4551fe4aae49ca9cf8fea6d46fe 100644 (file)
--- a/arch/m68k/include/asm/cacheflush_mm.h
+++ b/arch/m68k/include/asm/cacheflush_mm.h
@@ -191,6 +191,7 @@ extern void cache_push_v(unsigned long vaddr, int len);
 #define flush_cache_all() __flush_cache_all()
 
 #define flush_cache_vmap(start, end)           flush_cache_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_cache_all()
 
 static inline void flush_cache_mm(struct mm_struct *mm)

diff --git a/arch/mips/include/asm/cacheflush.h b/arch/mips/include/asm/cacheflush.h
index f36c2519ed9768b8eb570e9b8feb0f1ad0bf8de5..1f14132b3fc98afb6c44de0b7efda4d820978278 100644 (file)
--- a/arch/mips/include/asm/cacheflush.h
+++ b/arch/mips/include/asm/cacheflush.h
@@ -97,6 +97,8 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end)
                __flush_cache_vmap();
 }
 
+#define flush_cache_vmap_early(start, end)     do { } while (0)
+
 extern void (*__flush_cache_vunmap)(void);
 
 static inline void flush_cache_vunmap(unsigned long start, unsigned long end)

diff --git a/arch/nios2/include/asm/cacheflush.h b/arch/nios2/include/asm/cacheflush.h
index 348cea0977927a523022217bc3637b498a8e4185..81484a776b333a2d9c9b402461f296b3b091e219 100644 (file)
--- a/arch/nios2/include/asm/cacheflush.h
+++ b/arch/nios2/include/asm/cacheflush.h
@@ -38,6 +38,7 @@ void flush_icache_pages(struct vm_area_struct *vma, struct page *page,
 #define flush_icache_pages flush_icache_pages
 
 #define flush_cache_vmap(start, end)           flush_dcache_range(start, end)
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_dcache_range(start, end)
 
 extern void copy_to_user_page(struct vm_area_struct *vma, struct page *page,

diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h
index b4006f2a97052da67eaf3d5dcbac95197dc0fd69..ba4c05bc24d6901124deb89152c82a62aa3f5e4d 100644 (file)
--- a/arch/parisc/include/asm/cacheflush.h
+++ b/arch/parisc/include/asm/cacheflush.h
@@ -41,6 +41,7 @@ void flush_kernel_vmap_range(void *vaddr, int size);
 void invalidate_kernel_vmap_range(void *vaddr, int size);
 
 #define flush_cache_vmap(start, end)           flush_cache_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_cache_all()
 
 void flush_dcache_folio(struct folio *folio);

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index c729f60eaa00693cb3802e0f7201d45d8953f756..b549499eb3632c1da07b1a2b82b9d24b77e56eb5 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -416,7 +416,9 @@ config NUMA
        depends on SMP && MMU
        select ARCH_SUPPORTS_NUMA_BALANCING
        select GENERIC_ARCH_NUMA
+       select HAVE_SETUP_PER_CPU_AREA
        select NEED_PER_CPU_EMBED_FIRST_CHUNK
+       select NEED_PER_CPU_PAGE_FIRST_CHUNK
        select OF_NUMA
        select USE_PERCPU_NUMA_NODE_ID
        help

diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h
index 3cb53c4df27cfe4772f2f74623fc861f6a72360f..a129dac4521d35d69af22f713f5cdad27edf7f86 100644 (file)
--- a/arch/riscv/include/asm/cacheflush.h
+++ b/arch/riscv/include/asm/cacheflush.h
@@ -37,7 +37,8 @@ static inline void flush_dcache_page(struct page *page)
        flush_icache_mm(vma->vm_mm, 0)
 
 #ifdef CONFIG_64BIT
-#define flush_cache_vmap(start, end)   flush_tlb_kernel_range(start, end)
+#define flush_cache_vmap(start, end)           flush_tlb_kernel_range(start, end)
+#define flush_cache_vmap_early(start, end)     local_flush_tlb_kernel_range(start, end)
 #endif
 
 #ifndef CONFIG_SMP

diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index 8f3418c5f1724ba45e412ca52e0ef59ba0140638..a60416bbe19046a4aa2c303d6eb29d9bf1c0f2b6 100644 (file)
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -41,6 +41,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr);
 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
                     unsigned long end);
 void flush_tlb_kernel_range(unsigned long start, unsigned long end);
+void local_flush_tlb_kernel_range(unsigned long start, unsigned long end);
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,

diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c
index e962518530373da4492d6a9b5f27062232b61ece..c301c8d291d2df54f93b5579c3e56ed1461e4dc7 100644 (file)
--- a/arch/riscv/mm/kasan_init.c
+++ b/arch/riscv/mm/kasan_init.c
@@ -441,6 +441,14 @@ static void __init kasan_shallow_populate(void *start, void *end)
        kasan_shallow_populate_pgd(vaddr, vend);
 }
 
+#ifdef CONFIG_KASAN_VMALLOC
+void __init kasan_populate_early_vm_area_shadow(void *start, unsigned long size)
+{
+       kasan_populate(kasan_mem_to_shadow(start),
+                      kasan_mem_to_shadow(start + size));
+}
+#endif
+
 static void __init create_tmp_mapping(void)
 {
        void *ptr;

diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index e6659d7368b35403d1b91739080496bfc45442af..8aadc5f71c93bedf88c3618a531141de49d6b5a2 100644 (file)
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -66,6 +66,11 @@ static inline void local_flush_tlb_range_asid(unsigned long start,
                local_flush_tlb_range_threshold_asid(start, size, stride, asid);
 }
 
+void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+       local_flush_tlb_range_asid(start, end, PAGE_SIZE, FLUSH_TLB_NO_ASID);
+}
+
 static void __ipi_flush_tlb_all(void *info)
 {
        local_flush_tlb_all();

diff --git a/arch/sh/include/asm/cacheflush.h b/arch/sh/include/asm/cacheflush.h
index 878b6b551bd2d0119dd17f2918b73d3ae4120a77..51112f54552b329a307577a5a047f97172d56381 100644 (file)
--- a/arch/sh/include/asm/cacheflush.h
+++ b/arch/sh/include/asm/cacheflush.h
@@ -90,6 +90,7 @@ extern void copy_from_user_page(struct vm_area_struct *vma,
        unsigned long len);
 
 #define flush_cache_vmap(start, end)           local_flush_cache_all(NULL)
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         local_flush_cache_all(NULL)
 
 #define flush_dcache_mmap_lock(mapping)                do { } while (0)

diff --git a/arch/sparc/include/asm/cacheflush_32.h b/arch/sparc/include/asm/cacheflush_32.h
index f3b7270bf71b26ae7dcf77378d2b336363f307f5..9fee0ccfccb8e1b95a9d21ab293774fd6797eeb7 100644 (file)
--- a/arch/sparc/include/asm/cacheflush_32.h
+++ b/arch/sparc/include/asm/cacheflush_32.h
@@ -48,6 +48,7 @@ static inline void flush_dcache_page(struct page *page)
 #define flush_dcache_mmap_unlock(mapping)      do { } while (0)
 
 #define flush_cache_vmap(start, end)           flush_cache_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_cache_all()
 
 /* When a context switch happens we must flush all user windows so that

diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h
index 0e879004efff16e69afadb5867731960f20f9781..2b1261b77ecd1b9f93bdecbd1fa08a1985b08171 100644 (file)
--- a/arch/sparc/include/asm/cacheflush_64.h
+++ b/arch/sparc/include/asm/cacheflush_64.h
@@ -75,6 +75,7 @@ void flush_ptrace_access(struct vm_area_struct *, struct page *,
 #define flush_dcache_mmap_unlock(mapping)      do { } while (0)
 
 #define flush_cache_vmap(start, end)           do { } while (0)
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         do { } while (0)
 
 #endif /* !__ASSEMBLY__ */

diff --git a/arch/xtensa/include/asm/cacheflush.h b/arch/xtensa/include/asm/cacheflush.h
index 785a00ce83c11e8bbfa8e02b131315606060c35c..38bcecb0e457d9741c142cada4d38ec65ff0f88b 100644 (file)
--- a/arch/xtensa/include/asm/cacheflush.h
+++ b/arch/xtensa/include/asm/cacheflush.h
@@ -116,8 +116,9 @@ void flush_cache_page(struct vm_area_struct*,
 #define flush_cache_mm(mm)             flush_cache_all()
 #define flush_cache_dup_mm(mm)         flush_cache_mm(mm)
 
-#define flush_cache_vmap(start,end)    flush_cache_all()
-#define flush_cache_vunmap(start,end)  flush_cache_all()
+#define flush_cache_vmap(start,end)            flush_cache_all()
+#define flush_cache_vmap_early(start,end)      do { } while (0)
+#define flush_cache_vunmap(start,end)          flush_cache_all()
 
 void flush_dcache_folio(struct folio *folio);
 #define flush_dcache_folio flush_dcache_folio
@@ -140,6 +141,7 @@ void local_flush_cache_page(struct vm_area_struct *vma,
 #define flush_cache_dup_mm(mm)                         do { } while (0)
 
 #define flush_cache_vmap(start,end)                    do { } while (0)
+#define flush_cache_vmap_early(start,end)              do { } while (0)
 #define flush_cache_vunmap(start,end)                  do { } while (0)
 
 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0

diff --git a/include/asm-generic/cacheflush.h b/include/asm-generic/cacheflush.h
index 84ec53ccc450296f264cad5952b4cc26e73dad40..7ee8a179d1036e1d8010b8b18a8f3022e41c1695 100644 (file)
--- a/include/asm-generic/cacheflush.h
+++ b/include/asm-generic/cacheflush.h
@@ -91,6 +91,12 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end)
 }
 #endif
 
+#ifndef flush_cache_vmap_early
+static inline void flush_cache_vmap_early(unsigned long start, unsigned long end)
+{
+}
+#endif
+
 #ifndef flush_cache_vunmap
 static inline void flush_cache_vunmap(unsigned long start, unsigned long end)
 {

diff --git a/mm/percpu.c b/mm/percpu.c
index 7b97d31df76766f830c1da97427d54d81cea319b..4e11fc1e6deff0b2af1ee6be6b8de6bd1e7a18e2 100644 (file)
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -3333,13 +3333,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t
                if (rc < 0)
                        panic("failed to map percpu area, err=%d\n", rc);
 
-               /*
-                * FIXME: Archs with virtual cache should flush local
-                * cache for the linear mapping here - something
-                * equivalent to flush_cache_vmap() on the local cpu.
-                * flush_cache_vmap() can't be used as most supporting
-                * data structures are not set up yet.
-                */
+               flush_cache_vmap_early(unit_addr, unit_addr + ai->unit_size);
 
                /* copy static data */
                memcpy((void *)unit_addr, __per_cpu_load, ai->static_size);