vdso/datastore: Allocate data pages dynamically

Allocating the data pages as part of the kernel image does not work on
SPARC. The MMU will raise a fault when userspace tries to access them.

Allocate the data pages through the page allocator instead.

Unused pages in the vDSO VMA are still allocated to keep the virtual
addresses aligned. Switch the mapping from PFNs to 'struct page' as that is
required for dynamically allocated pages.  This also aligns the allocation
of the datapages with the code pages and is a prerequisite for mlockall()
support.

VM_MIXEDMAP is necessary for the call to vmf_insert_page() in the timens
prefault path to work.

The data pages need to be order-0, non-compound pages so that the mapping
to userspace and the different orderings work.

These pages are also used by the timekeeping, random pool and architecture
initialization code. Some of these are running before the page allocator is
available. To keep these subsytems working without changes, introduce
early, statically data storage which will then replaced by the real one as
soon as that is available.

Signed-off-by: Thomas Weißschuh <thomas.weissschuh@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Reviewed-by: Christophe Leroy (CS GROUP) <chleroy@kernel.org>
Link: https://patch.msgid.link/20260304-vdso-sparc64-generic-2-v6-3-d8eb3b0e1410@linutronix.de

diff --git a/include/linux/vdso_datastore.h b/include/linux/vdso_datastore.h
index a91fa24b06e09321fdff8c2c7bdfbc1b206db574..0b530428db711e58660e797d9d3cf5dce60217fe 100644 (file)
--- a/include/linux/vdso_datastore.h
+++ b/include/linux/vdso_datastore.h
@@ -2,9 +2,15 @@
 #ifndef _LINUX_VDSO_DATASTORE_H
 #define _LINUX_VDSO_DATASTORE_H
 
+#ifdef CONFIG_HAVE_GENERIC_VDSO
 #include <linux/mm_types.h>
 
 extern const struct vm_special_mapping vdso_vvar_mapping;
 struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
 
+void __init vdso_setup_data_pages(void);
+#else /* !CONFIG_HAVE_GENERIC_VDSO */
+static inline void vdso_setup_data_pages(void) { }
+#endif /* CONFIG_HAVE_GENERIC_VDSO */
+
 #endif /* _LINUX_VDSO_DATASTORE_H */

diff --git a/init/main.c b/init/main.c
index 1cb395dd94e43f3e0e3f75dabb0f5e2737f8e88a..de867b2693d215a8e2cf67cf183d87b4272196d0 100644 (file)
--- a/init/main.c
+++ b/init/main.c
@@ -105,6 +105,7 @@
 #include <linux/ptdump.h>
 #include <linux/time_namespace.h>
 #include <linux/unaligned.h>
+#include <linux/vdso_datastore.h>
 #include <net/net_namespace.h>
 
 #include <asm/io.h>
@@ -1119,6 +1120,7 @@ void start_kernel(void)
        srcu_init();
        hrtimers_init();
        softirq_init();
+       vdso_setup_data_pages();
        timekeeping_init();
        time_init();
 

diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c
index 7377fcb6e1dfe31d02ffcae371fdf9da069ae4c1..faebf5b7cd6ecc39062a5153e0ecd886497fddc3 100644 (file)
--- a/lib/vdso/datastore.c
+++ b/lib/vdso/datastore.c
@@ -1,52 +1,79 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
-#include <linux/linkage.h>
+#include <linux/gfp.h>
+#include <linux/init.h>
 #include <linux/mm.h>
 #include <linux/time_namespace.h>
 #include <linux/types.h>
 #include <linux/vdso_datastore.h>
 #include <vdso/datapage.h>
 
-/*
- * The vDSO data page.
- */
+static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {};
+
 #ifdef CONFIG_GENERIC_GETTIMEOFDAY
-static union {
-       struct vdso_time_data   data;
-       u8                      page[PAGE_SIZE];
-} vdso_time_data_store __page_aligned_data;
-struct vdso_time_data *vdso_k_time_data = &vdso_time_data_store.data;
-static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE);
+struct vdso_time_data *vdso_k_time_data __refdata =
+       (void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE];
+
+static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE);
 #endif /* CONFIG_GENERIC_GETTIMEOFDAY */
 
 #ifdef CONFIG_VDSO_GETRANDOM
-static union {
-       struct vdso_rng_data    data;
-       u8                      page[PAGE_SIZE];
-} vdso_rng_data_store __page_aligned_data;
-struct vdso_rng_data *vdso_k_rng_data = &vdso_rng_data_store.data;
-static_assert(sizeof(vdso_rng_data_store) == PAGE_SIZE);
+struct vdso_rng_data *vdso_k_rng_data __refdata =
+       (void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE];
+
+static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE);
 #endif /* CONFIG_VDSO_GETRANDOM */
 
 #ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
-static union {
-       struct vdso_arch_data   data;
-       u8                      page[VDSO_ARCH_DATA_SIZE];
-} vdso_arch_data_store __page_aligned_data;
-struct vdso_arch_data *vdso_k_arch_data = &vdso_arch_data_store.data;
+struct vdso_arch_data *vdso_k_arch_data __refdata =
+       (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
 #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
 
+void __init vdso_setup_data_pages(void)
+{
+       unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE);
+       struct page *pages;
+
+       /*
+        * Allocate the data pages dynamically. SPARC does not support mapping
+        * static pages to be mapped into userspace.
+        * It is also a requirement for mlockall() support.
+        *
+        * Do not use folios. In time namespaces the pages are mapped in a different order
+        * to userspace, which is not handled by the folio optimizations in finish_fault().
+        */
+       pages = alloc_pages(GFP_KERNEL, order);
+       if (!pages)
+               panic("Unable to allocate VDSO storage pages");
+
+       /* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */
+       split_page(pages, order);
+
+       /* Move the data already written by other subsystems to the new pages */
+       memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE);
+
+       if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
+               vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET);
+
+       if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
+               vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET);
+
+       if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
+               vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START);
+}
+
 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
                             struct vm_area_struct *vma, struct vm_fault *vmf)
 {
-       struct page *timens_page = find_timens_vvar_page(vma);
-       unsigned long pfn;
+       struct page *page, *timens_page;
+
+       timens_page = find_timens_vvar_page(vma);
 
        switch (vmf->pgoff) {
        case VDSO_TIME_PAGE_OFFSET:
                if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
                        return VM_FAULT_SIGBUS;
-               pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
+               page = virt_to_page(vdso_k_time_data);
                if (timens_page) {
                        /*
                         * Fault in VVAR page too, since it will be accessed
@@ -56,10 +83,10 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
                        vm_fault_t err;
 
                        addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
-                       err = vmf_insert_pfn(vma, addr, pfn);
+                       err = vmf_insert_page(vma, addr, page);
                        if (unlikely(err & VM_FAULT_ERROR))
                                return err;
-                       pfn = page_to_pfn(timens_page);
+                       page = timens_page;
                }
                break;
        case VDSO_TIMENS_PAGE_OFFSET:
@@ -72,24 +99,25 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
                 */
                if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
                        return VM_FAULT_SIGBUS;
-               pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
+               page = virt_to_page(vdso_k_time_data);
                break;
        case VDSO_RNG_PAGE_OFFSET:
                if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM))
                        return VM_FAULT_SIGBUS;
-               pfn = __phys_to_pfn(__pa_symbol(vdso_k_rng_data));
+               page = virt_to_page(vdso_k_rng_data);
                break;
        case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END:
                if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
                        return VM_FAULT_SIGBUS;
-               pfn = __phys_to_pfn(__pa_symbol(vdso_k_arch_data)) +
-                       vmf->pgoff - VDSO_ARCH_PAGES_START;
+               page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START;
                break;
        default:
                return VM_FAULT_SIGBUS;
        }
 
-       return vmf_insert_pfn(vma, vmf->address, pfn);
+       get_page(page);
+       vmf->page = page;
+       return 0;
 }
 
 const struct vm_special_mapping vdso_vvar_mapping = {
@@ -101,7 +129,7 @@ struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned
 {
        return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
                                        VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP |
-                                       VM_PFNMAP | VM_SEALED_SYSMAP,
+                                       VM_MIXEDMAP | VM_SEALED_SYSMAP,
                                        &vdso_vvar_mapping);
 }