arch/arm64/kernel/vdso.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VDSO implementations.
   4  *
   5  * Copyright (C) 2012 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  */
   9
  10 #include <linux/cache.h>
  11 #include <linux/clocksource.h>
  12 #include <linux/elf.h>
  13 #include <linux/err.h>
  14 #include <linux/errno.h>
  15 #include <linux/gfp.h>
  16 #include <linux/kernel.h>
  17 #include <linux/mm.h>
  18 #include <linux/sched.h>
  19 #include <linux/signal.h>
  20 #include <linux/slab.h>
  21 #include <linux/timekeeper_internal.h>
  22 #include <linux/vmalloc.h>
  23
  24 #include <asm/cacheflush.h>
  25 #include <asm/signal32.h>
  26 #include <asm/vdso.h>
  27 #include <asm/vdso_datapage.h>
  28
  29 extern char vdso_start[], vdso_end[];
  30 static unsigned long vdso_pages __ro_after_init;
  31
  32 /*
  33  * The vDSO data page.
  34  */
  35 static union {
  36         struct vdso_data        data;
  37         u8                      page[PAGE_SIZE];
  38 } vdso_data_store __page_aligned_data;
  39 struct vdso_data *vdso_data = &vdso_data_store.data;
  40
  41 #ifdef CONFIG_COMPAT
  42 /*
  43  * Create and map the vectors page for AArch32 tasks.
  44  */
  45 #define C_VECTORS       0
  46 #define C_SIGPAGE       1
  47 #define C_PAGES         (C_SIGPAGE + 1)
  48 static struct page *aarch32_vdso_pages[C_PAGES] __ro_after_init;
  49 static const struct vm_special_mapping aarch32_vdso_spec[C_PAGES] = {
  50         {
  51                 .name   = "[vectors]", /* ABI */
  52                 .pages  = &aarch32_vdso_pages[C_VECTORS],
  53         },
  54         {
  55                 .name   = "[sigpage]", /* ABI */
  56                 .pages  = &aarch32_vdso_pages[C_SIGPAGE],
  57         },
  58 };
  59
  60 static int aarch32_alloc_kuser_vdso_page(void)
  61 {
  62         extern char __kuser_helper_start[], __kuser_helper_end[];
  63         int kuser_sz = __kuser_helper_end - __kuser_helper_start;
  64         unsigned long vdso_page;
  65
  66         if (!IS_ENABLED(CONFIG_KUSER_HELPERS))
  67                 return 0;
  68
  69         vdso_page = get_zeroed_page(GFP_ATOMIC);
  70         if (!vdso_page)
  71                 return -ENOMEM;
  72
  73         memcpy((void *)(vdso_page + 0x1000 - kuser_sz), __kuser_helper_start,
  74                kuser_sz);
  75         aarch32_vdso_pages[C_VECTORS] = virt_to_page(vdso_page);
  76         flush_dcache_page(aarch32_vdso_pages[C_VECTORS]);
  77         return 0;
  78 }
  79
  80 static int __init aarch32_alloc_vdso_pages(void)
  81 {
  82         extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
  83         int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start;
  84         unsigned long sigpage;
  85         int ret;
  86
  87         sigpage = get_zeroed_page(GFP_ATOMIC);
  88         if (!sigpage)
  89                 return -ENOMEM;
  90
  91         memcpy((void *)sigpage, __aarch32_sigret_code_start, sigret_sz);
  92         aarch32_vdso_pages[C_SIGPAGE] = virt_to_page(sigpage);
  93         flush_dcache_page(aarch32_vdso_pages[C_SIGPAGE]);
  94
  95         ret = aarch32_alloc_kuser_vdso_page();
  96         if (ret)
  97                 free_page(sigpage);
  98
  99         return ret;
 100 }
 101 arch_initcall(aarch32_alloc_vdso_pages);
 102
 103 static int aarch32_kuser_helpers_setup(struct mm_struct *mm)
 104 {
 105         void *ret;
 106
 107         if (!IS_ENABLED(CONFIG_KUSER_HELPERS))
 108                 return 0;
 109
 110         /*
 111          * Avoid VM_MAYWRITE for compatibility with arch/arm/, where it's
 112          * not safe to CoW the page containing the CPU exception vectors.
 113          */
 114         ret = _install_special_mapping(mm, AARCH32_VECTORS_BASE, PAGE_SIZE,
 115                                        VM_READ | VM_EXEC |
 116                                        VM_MAYREAD | VM_MAYEXEC,
 117                                        &aarch32_vdso_spec[C_VECTORS]);
 118
 119         return PTR_ERR_OR_ZERO(ret);
 120 }
 121
 122 static int aarch32_sigreturn_setup(struct mm_struct *mm)
 123 {
 124         unsigned long addr;
 125         void *ret;
 126
 127         addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0);
 128         if (IS_ERR_VALUE(addr)) {
 129                 ret = ERR_PTR(addr);
 130                 goto out;
 131         }
 132
 133         /*
 134          * VM_MAYWRITE is required to allow gdb to Copy-on-Write and
 135          * set breakpoints.
 136          */
 137         ret = _install_special_mapping(mm, addr, PAGE_SIZE,
 138                                        VM_READ | VM_EXEC | VM_MAYREAD |
 139                                        VM_MAYWRITE | VM_MAYEXEC,
 140                                        &aarch32_vdso_spec[C_SIGPAGE]);
 141         if (IS_ERR(ret))
 142                 goto out;
 143
 144         mm->context.vdso = (void *)addr;
 145
 146 out:
 147         return PTR_ERR_OR_ZERO(ret);
 148 }
 149
 150 int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 151 {
 152         struct mm_struct *mm = current->mm;
 153         int ret;
 154
 155         if (down_write_killable(&mm->mmap_sem))
 156                 return -EINTR;
 157
 158         ret = aarch32_kuser_helpers_setup(mm);
 159         if (ret)
 160                 goto out;
 161
 162         ret = aarch32_sigreturn_setup(mm);
 163
 164 out:
 165         up_write(&mm->mmap_sem);
 166         return ret;
 167 }
 168 #endif /* CONFIG_COMPAT */
 169
 170 static int vdso_mremap(const struct vm_special_mapping *sm,
 171                 struct vm_area_struct *new_vma)
 172 {
 173         unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
 174         unsigned long vdso_size = vdso_end - vdso_start;
 175
 176         if (vdso_size != new_size)
 177                 return -EINVAL;
 178
 179         current->mm->context.vdso = (void *)new_vma->vm_start;
 180
 181         return 0;
 182 }
 183
 184 static struct vm_special_mapping vdso_spec[2] __ro_after_init = {
 185         {
 186                 .name   = "[vvar]",
 187         },
 188         {
 189                 .name   = "[vdso]",
 190                 .mremap = vdso_mremap,
 191         },
 192 };
 193
 194 static int __init vdso_init(void)
 195 {
 196         int i;
 197         struct page **vdso_pagelist;
 198         unsigned long pfn;
 199
 200         if (memcmp(vdso_start, "\177ELF", 4)) {
 201                 pr_err("vDSO is not a valid ELF object!\n");
 202                 return -EINVAL;
 203         }
 204
 205         vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT;
 206
 207         /* Allocate the vDSO pagelist, plus a page for the data. */
 208         vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *),
 209                                 GFP_KERNEL);
 210         if (vdso_pagelist == NULL)
 211                 return -ENOMEM;
 212
 213         /* Grab the vDSO data page. */
 214         vdso_pagelist[0] = phys_to_page(__pa_symbol(vdso_data));
 215
 216
 217         /* Grab the vDSO code pages. */
 218         pfn = sym_to_pfn(vdso_start);
 219
 220         for (i = 0; i < vdso_pages; i++)
 221                 vdso_pagelist[i + 1] = pfn_to_page(pfn + i);
 222
 223         vdso_spec[0].pages = &vdso_pagelist[0];
 224         vdso_spec[1].pages = &vdso_pagelist[1];
 225
 226         return 0;
 227 }
 228 arch_initcall(vdso_init);
 229
 230 int arch_setup_additional_pages(struct linux_binprm *bprm,
 231                                 int uses_interp)
 232 {
 233         struct mm_struct *mm = current->mm;
 234         unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
 235         void *ret;
 236
 237         vdso_text_len = vdso_pages << PAGE_SHIFT;
 238         /* Be sure to map the data page */
 239         vdso_mapping_len = vdso_text_len + PAGE_SIZE;
 240
 241         if (down_write_killable(&mm->mmap_sem))
 242                 return -EINTR;
 243         vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
 244         if (IS_ERR_VALUE(vdso_base)) {
 245                 ret = ERR_PTR(vdso_base);
 246                 goto up_fail;
 247         }
 248         ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE,
 249                                        VM_READ|VM_MAYREAD,
 250                                        &vdso_spec[0]);
 251         if (IS_ERR(ret))
 252                 goto up_fail;
 253
 254         vdso_base += PAGE_SIZE;
 255         mm->context.vdso = (void *)vdso_base;
 256         ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
 257                                        VM_READ|VM_EXEC|
 258                                        VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
 259                                        &vdso_spec[1]);
 260         if (IS_ERR(ret))
 261                 goto up_fail;
 262
 263
 264         up_write(&mm->mmap_sem);
 265         return 0;
 266
 267 up_fail:
 268         mm->context.vdso = NULL;
 269         up_write(&mm->mmap_sem);
 270         return PTR_ERR(ret);
 271 }
 272
 273 /*
 274  * Update the vDSO data page to keep in sync with kernel timekeeping.
 275  */
 276 void update_vsyscall(struct timekeeper *tk)
 277 {
 278         u32 use_syscall = !tk->tkr_mono.clock->archdata.vdso_direct;
 279
 280         ++vdso_data->tb_seq_count;
 281         smp_wmb();
 282
 283         vdso_data->use_syscall                  = use_syscall;
 284         vdso_data->xtime_coarse_sec             = tk->xtime_sec;
 285         vdso_data->xtime_coarse_nsec            = tk->tkr_mono.xtime_nsec >>
 286                                                         tk->tkr_mono.shift;
 287         vdso_data->wtm_clock_sec                = tk->wall_to_monotonic.tv_sec;
 288         vdso_data->wtm_clock_nsec               = tk->wall_to_monotonic.tv_nsec;
 289
 290         /* Read without the seqlock held by clock_getres() */
 291         WRITE_ONCE(vdso_data->hrtimer_res, hrtimer_resolution);
 292
 293         if (!use_syscall) {
 294                 /* tkr_mono.cycle_last == tkr_raw.cycle_last */
 295                 vdso_data->cs_cycle_last        = tk->tkr_mono.cycle_last;
 296                 vdso_data->raw_time_sec         = tk->raw_sec;
 297                 vdso_data->raw_time_nsec        = tk->tkr_raw.xtime_nsec;
 298                 vdso_data->xtime_clock_sec      = tk->xtime_sec;
 299                 vdso_data->xtime_clock_nsec     = tk->tkr_mono.xtime_nsec;
 300                 vdso_data->cs_mono_mult         = tk->tkr_mono.mult;
 301                 vdso_data->cs_raw_mult          = tk->tkr_raw.mult;
 302                 /* tkr_mono.shift == tkr_raw.shift */
 303                 vdso_data->cs_shift             = tk->tkr_mono.shift;
 304         }
 305
 306         smp_wmb();
 307         ++vdso_data->tb_seq_count;
 308 }
 309
 310 void update_vsyscall_tz(void)
 311 {
 312         vdso_data->tz_minuteswest       = sys_tz.tz_minuteswest;
 313         vdso_data->tz_dsttime           = sys_tz.tz_dsttime;
 314 }