--- /dev/null
+From 37868fe113ff2ba814b3b4eb12df214df555f8dc Mon Sep 17 00:00:00 2001
+From: Andy Lutomirski <luto@kernel.org>
+Date: Thu, 30 Jul 2015 14:31:32 -0700
+Subject: x86/ldt: Make modify_ldt synchronous
+
+From: Andy Lutomirski <luto@kernel.org>
+
+commit 37868fe113ff2ba814b3b4eb12df214df555f8dc upstream.
+
+modify_ldt() has questionable locking and does not synchronize
+threads. Improve it: redesign the locking and synchronize all
+threads' LDTs using an IPI on all modifications.
+
+This will dramatically slow down modify_ldt in multithreaded
+programs, but there shouldn't be any multithreaded programs that
+care about modify_ldt's performance in the first place.
+
+This fixes some fallout from the CVE-2015-5157 fixes.
+
+Signed-off-by: Andy Lutomirski <luto@kernel.org>
+Reviewed-by: Borislav Petkov <bp@suse.de>
+Cc: Andrew Cooper <andrew.cooper3@citrix.com>
+Cc: Andy Lutomirski <luto@amacapital.net>
+Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
+Cc: Borislav Petkov <bp@alien8.de>
+Cc: Brian Gerst <brgerst@gmail.com>
+Cc: Denys Vlasenko <dvlasenk@redhat.com>
+Cc: H. Peter Anvin <hpa@zytor.com>
+Cc: Jan Beulich <jbeulich@suse.com>
+Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Sasha Levin <sasha.levin@oracle.com>
+Cc: Steven Rostedt <rostedt@goodmis.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Cc: security@kernel.org <security@kernel.org>
+Cc: xen-devel <xen-devel@lists.xen.org>
+Link: http://lkml.kernel.org/r/4c6978476782160600471bd865b318db34c7b628.1438291540.git.luto@kernel.org
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+---
+ arch/x86/include/asm/desc.h | 15 --
+ arch/x86/include/asm/mmu.h | 3
+ arch/x86/include/asm/mmu_context.h | 54 ++++++-
+ arch/x86/kernel/cpu/common.c | 4
+ arch/x86/kernel/cpu/perf_event.c | 12 +
+ arch/x86/kernel/ldt.c | 264 ++++++++++++++++++++-----------------
+ arch/x86/kernel/process_64.c | 4
+ arch/x86/kernel/step.c | 6
+ arch/x86/power/cpu.c | 3
+ 9 files changed, 211 insertions(+), 154 deletions(-)
+
+--- a/arch/x86/include/asm/desc.h
++++ b/arch/x86/include/asm/desc.h
+@@ -280,21 +280,6 @@ static inline void clear_LDT(void)
+ set_ldt(NULL, 0);
+ }
+
+-/*
+- * load one particular LDT into the current CPU
+- */
+-static inline void load_LDT_nolock(mm_context_t *pc)
+-{
+- set_ldt(pc->ldt, pc->size);
+-}
+-
+-static inline void load_LDT(mm_context_t *pc)
+-{
+- preempt_disable();
+- load_LDT_nolock(pc);
+- preempt_enable();
+-}
+-
+ static inline unsigned long get_desc_base(const struct desc_struct *desc)
+ {
+ return (unsigned)(desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24));
+--- a/arch/x86/include/asm/mmu.h
++++ b/arch/x86/include/asm/mmu.h
+@@ -9,8 +9,7 @@
+ * we put the segment information here.
+ */
+ typedef struct {
+- void *ldt;
+- int size;
++ struct ldt_struct *ldt;
+
+ #ifdef CONFIG_X86_64
+ /* True if mm supports a task running in 32 bit compatibility mode. */
+--- a/arch/x86/include/asm/mmu_context.h
++++ b/arch/x86/include/asm/mmu_context.h
+@@ -34,6 +34,50 @@ static inline void load_mm_cr4(struct mm
+ #endif
+
+ /*
++ * ldt_structs can be allocated, used, and freed, but they are never
++ * modified while live.
++ */
++struct ldt_struct {
++ /*
++ * Xen requires page-aligned LDTs with special permissions. This is
++ * needed to prevent us from installing evil descriptors such as
++ * call gates. On native, we could merge the ldt_struct and LDT
++ * allocations, but it's not worth trying to optimize.
++ */
++ struct desc_struct *entries;
++ int size;
++};
++
++static inline void load_mm_ldt(struct mm_struct *mm)
++{
++ struct ldt_struct *ldt;
++
++ /* lockless_dereference synchronizes with smp_store_release */
++ ldt = lockless_dereference(mm->context.ldt);
++
++ /*
++ * Any change to mm->context.ldt is followed by an IPI to all
++ * CPUs with the mm active. The LDT will not be freed until
++ * after the IPI is handled by all such CPUs. This means that,
++ * if the ldt_struct changes before we return, the values we see
++ * will be safe, and the new values will be loaded before we run
++ * any user code.
++ *
++ * NB: don't try to convert this to use RCU without extreme care.
++ * We would still need IRQs off, because we don't want to change
++ * the local LDT after an IPI loaded a newer value than the one
++ * that we can see.
++ */
++
++ if (unlikely(ldt))
++ set_ldt(ldt->entries, ldt->size);
++ else
++ clear_LDT();
++
++ DEBUG_LOCKS_WARN_ON(preemptible());
++}
++
++/*
+ * Used for LDT copy/destruction.
+ */
+ int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
+@@ -78,12 +122,12 @@ static inline void switch_mm(struct mm_s
+ * was called and then modify_ldt changed
+ * prev->context.ldt but suppressed an IPI to this CPU.
+ * In this case, prev->context.ldt != NULL, because we
+- * never free an LDT while the mm still exists. That
+- * means that next->context.ldt != prev->context.ldt,
+- * because mms never share an LDT.
++ * never set context.ldt to NULL while the mm still
++ * exists. That means that next->context.ldt !=
++ * prev->context.ldt, because mms never share an LDT.
+ */
+ if (unlikely(prev->context.ldt != next->context.ldt))
+- load_LDT_nolock(&next->context);
++ load_mm_ldt(next);
+ }
+ #ifdef CONFIG_SMP
+ else {
+@@ -106,7 +150,7 @@ static inline void switch_mm(struct mm_s
+ load_cr3(next->pgd);
+ trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
+ load_mm_cr4(next);
+- load_LDT_nolock(&next->context);
++ load_mm_ldt(next);
+ }
+ }
+ #endif
+--- a/arch/x86/kernel/cpu/common.c
++++ b/arch/x86/kernel/cpu/common.c
+@@ -1434,7 +1434,7 @@ void cpu_init(void)
+ load_sp0(t, ¤t->thread);
+ set_tss_desc(cpu, t);
+ load_TR_desc();
+- load_LDT(&init_mm.context);
++ load_mm_ldt(&init_mm);
+
+ clear_all_debug_regs();
+ dbg_restore_debug_regs();
+@@ -1483,7 +1483,7 @@ void cpu_init(void)
+ load_sp0(t, thread);
+ set_tss_desc(cpu, t);
+ load_TR_desc();
+- load_LDT(&init_mm.context);
++ load_mm_ldt(&init_mm);
+
+ t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
+
+--- a/arch/x86/kernel/cpu/perf_event.c
++++ b/arch/x86/kernel/cpu/perf_event.c
+@@ -2170,21 +2170,25 @@ static unsigned long get_segment_base(un
+ int idx = segment >> 3;
+
+ if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
++ struct ldt_struct *ldt;
++
+ if (idx > LDT_ENTRIES)
+ return 0;
+
+- if (idx > current->active_mm->context.size)
++ /* IRQs are off, so this synchronizes with smp_store_release */
++ ldt = lockless_dereference(current->active_mm->context.ldt);
++ if (!ldt || idx > ldt->size)
+ return 0;
+
+- desc = current->active_mm->context.ldt;
++ desc = &ldt->entries[idx];
+ } else {
+ if (idx > GDT_ENTRIES)
+ return 0;
+
+- desc = raw_cpu_ptr(gdt_page.gdt);
++ desc = raw_cpu_ptr(gdt_page.gdt) + idx;
+ }
+
+- return get_desc_base(desc + idx);
++ return get_desc_base(desc);
+ }
+
+ #ifdef CONFIG_COMPAT
+--- a/arch/x86/kernel/ldt.c
++++ b/arch/x86/kernel/ldt.c
+@@ -12,6 +12,7 @@
+ #include <linux/string.h>
+ #include <linux/mm.h>
+ #include <linux/smp.h>
++#include <linux/slab.h>
+ #include <linux/vmalloc.h>
+ #include <linux/uaccess.h>
+
+@@ -20,82 +21,82 @@
+ #include <asm/mmu_context.h>
+ #include <asm/syscalls.h>
+
+-#ifdef CONFIG_SMP
++/* context.lock is held for us, so we don't need any locking. */
+ static void flush_ldt(void *current_mm)
+ {
+- if (current->active_mm == current_mm)
+- load_LDT(¤t->active_mm->context);
++ mm_context_t *pc;
++
++ if (current->active_mm != current_mm)
++ return;
++
++ pc = ¤t->active_mm->context;
++ set_ldt(pc->ldt->entries, pc->ldt->size);
+ }
+-#endif
+
+-static int alloc_ldt(mm_context_t *pc, int mincount, int reload)
++/* The caller must call finalize_ldt_struct on the result. LDT starts zeroed. */
++static struct ldt_struct *alloc_ldt_struct(int size)
+ {
+- void *oldldt, *newldt;
+- int oldsize;
++ struct ldt_struct *new_ldt;
++ int alloc_size;
+
+- if (mincount <= pc->size)
+- return 0;
+- oldsize = pc->size;
+- mincount = (mincount + (PAGE_SIZE / LDT_ENTRY_SIZE - 1)) &
+- (~(PAGE_SIZE / LDT_ENTRY_SIZE - 1));
+- if (mincount * LDT_ENTRY_SIZE > PAGE_SIZE)
+- newldt = vmalloc(mincount * LDT_ENTRY_SIZE);
+- else
+- newldt = (void *)__get_free_page(GFP_KERNEL);
++ if (size > LDT_ENTRIES)
++ return NULL;
+
+- if (!newldt)
+- return -ENOMEM;
++ new_ldt = kmalloc(sizeof(struct ldt_struct), GFP_KERNEL);
++ if (!new_ldt)
++ return NULL;
++
++ BUILD_BUG_ON(LDT_ENTRY_SIZE != sizeof(struct desc_struct));
++ alloc_size = size * LDT_ENTRY_SIZE;
++
++ /*
++ * Xen is very picky: it requires a page-aligned LDT that has no
++ * trailing nonzero bytes in any page that contains LDT descriptors.
++ * Keep it simple: zero the whole allocation and never allocate less
++ * than PAGE_SIZE.
++ */
++ if (alloc_size > PAGE_SIZE)
++ new_ldt->entries = vzalloc(alloc_size);
++ else
++ new_ldt->entries = kzalloc(PAGE_SIZE, GFP_KERNEL);
+
+- if (oldsize)
+- memcpy(newldt, pc->ldt, oldsize * LDT_ENTRY_SIZE);
+- oldldt = pc->ldt;
+- memset(newldt + oldsize * LDT_ENTRY_SIZE, 0,
+- (mincount - oldsize) * LDT_ENTRY_SIZE);
+-
+- paravirt_alloc_ldt(newldt, mincount);
+-
+-#ifdef CONFIG_X86_64
+- /* CHECKME: Do we really need this ? */
+- wmb();
+-#endif
+- pc->ldt = newldt;
+- wmb();
+- pc->size = mincount;
+- wmb();
+-
+- if (reload) {
+-#ifdef CONFIG_SMP
+- preempt_disable();
+- load_LDT(pc);
+- if (!cpumask_equal(mm_cpumask(current->mm),
+- cpumask_of(smp_processor_id())))
+- smp_call_function(flush_ldt, current->mm, 1);
+- preempt_enable();
+-#else
+- load_LDT(pc);
+-#endif
+- }
+- if (oldsize) {
+- paravirt_free_ldt(oldldt, oldsize);
+- if (oldsize * LDT_ENTRY_SIZE > PAGE_SIZE)
+- vfree(oldldt);
+- else
+- put_page(virt_to_page(oldldt));
++ if (!new_ldt->entries) {
++ kfree(new_ldt);
++ return NULL;
+ }
+- return 0;
++
++ new_ldt->size = size;
++ return new_ldt;
+ }
+
+-static inline int copy_ldt(mm_context_t *new, mm_context_t *old)
++/* After calling this, the LDT is immutable. */
++static void finalize_ldt_struct(struct ldt_struct *ldt)
+ {
+- int err = alloc_ldt(new, old->size, 0);
+- int i;
++ paravirt_alloc_ldt(ldt->entries, ldt->size);
++}
++
++/* context.lock is held */
++static void install_ldt(struct mm_struct *current_mm,
++ struct ldt_struct *ldt)
++{
++ /* Synchronizes with lockless_dereference in load_mm_ldt. */
++ smp_store_release(¤t_mm->context.ldt, ldt);
++
++ /* Activate the LDT for all CPUs using current_mm. */
++ on_each_cpu_mask(mm_cpumask(current_mm), flush_ldt, current_mm, true);
++}
+
+- if (err < 0)
+- return err;
++static void free_ldt_struct(struct ldt_struct *ldt)
++{
++ if (likely(!ldt))
++ return;
+
+- for (i = 0; i < old->size; i++)
+- write_ldt_entry(new->ldt, i, old->ldt + i * LDT_ENTRY_SIZE);
+- return 0;
++ paravirt_free_ldt(ldt->entries, ldt->size);
++ if (ldt->size * LDT_ENTRY_SIZE > PAGE_SIZE)
++ vfree(ldt->entries);
++ else
++ kfree(ldt->entries);
++ kfree(ldt);
+ }
+
+ /*
+@@ -104,17 +105,37 @@ static inline int copy_ldt(mm_context_t
+ */
+ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
+ {
++ struct ldt_struct *new_ldt;
+ struct mm_struct *old_mm;
+ int retval = 0;
+
+ mutex_init(&mm->context.lock);
+- mm->context.size = 0;
+ old_mm = current->mm;
+- if (old_mm && old_mm->context.size > 0) {
+- mutex_lock(&old_mm->context.lock);
+- retval = copy_ldt(&mm->context, &old_mm->context);
+- mutex_unlock(&old_mm->context.lock);
++ if (!old_mm) {
++ mm->context.ldt = NULL;
++ return 0;
+ }
++
++ mutex_lock(&old_mm->context.lock);
++ if (!old_mm->context.ldt) {
++ mm->context.ldt = NULL;
++ goto out_unlock;
++ }
++
++ new_ldt = alloc_ldt_struct(old_mm->context.ldt->size);
++ if (!new_ldt) {
++ retval = -ENOMEM;
++ goto out_unlock;
++ }
++
++ memcpy(new_ldt->entries, old_mm->context.ldt->entries,
++ new_ldt->size * LDT_ENTRY_SIZE);
++ finalize_ldt_struct(new_ldt);
++
++ mm->context.ldt = new_ldt;
++
++out_unlock:
++ mutex_unlock(&old_mm->context.lock);
+ return retval;
+ }
+
+@@ -125,53 +146,47 @@ int init_new_context(struct task_struct
+ */
+ void destroy_context(struct mm_struct *mm)
+ {
+- if (mm->context.size) {
+-#ifdef CONFIG_X86_32
+- /* CHECKME: Can this ever happen ? */
+- if (mm == current->active_mm)
+- clear_LDT();
+-#endif
+- paravirt_free_ldt(mm->context.ldt, mm->context.size);
+- if (mm->context.size * LDT_ENTRY_SIZE > PAGE_SIZE)
+- vfree(mm->context.ldt);
+- else
+- put_page(virt_to_page(mm->context.ldt));
+- mm->context.size = 0;
+- }
++ free_ldt_struct(mm->context.ldt);
++ mm->context.ldt = NULL;
+ }
+
+ static int read_ldt(void __user *ptr, unsigned long bytecount)
+ {
+- int err;
++ int retval;
+ unsigned long size;
+ struct mm_struct *mm = current->mm;
+
+- if (!mm->context.size)
+- return 0;
++ mutex_lock(&mm->context.lock);
++
++ if (!mm->context.ldt) {
++ retval = 0;
++ goto out_unlock;
++ }
++
+ if (bytecount > LDT_ENTRY_SIZE * LDT_ENTRIES)
+ bytecount = LDT_ENTRY_SIZE * LDT_ENTRIES;
+
+- mutex_lock(&mm->context.lock);
+- size = mm->context.size * LDT_ENTRY_SIZE;
++ size = mm->context.ldt->size * LDT_ENTRY_SIZE;
+ if (size > bytecount)
+ size = bytecount;
+
+- err = 0;
+- if (copy_to_user(ptr, mm->context.ldt, size))
+- err = -EFAULT;
+- mutex_unlock(&mm->context.lock);
+- if (err < 0)
+- goto error_return;
++ if (copy_to_user(ptr, mm->context.ldt->entries, size)) {
++ retval = -EFAULT;
++ goto out_unlock;
++ }
++
+ if (size != bytecount) {
+- /* zero-fill the rest */
+- if (clear_user(ptr + size, bytecount - size) != 0) {
+- err = -EFAULT;
+- goto error_return;
++ /* Zero-fill the rest and pretend we read bytecount bytes. */
++ if (clear_user(ptr + size, bytecount - size)) {
++ retval = -EFAULT;
++ goto out_unlock;
+ }
+ }
+- return bytecount;
+-error_return:
+- return err;
++ retval = bytecount;
++
++out_unlock:
++ mutex_unlock(&mm->context.lock);
++ return retval;
+ }
+
+ static int read_default_ldt(void __user *ptr, unsigned long bytecount)
+@@ -195,6 +210,8 @@ static int write_ldt(void __user *ptr, u
+ struct desc_struct ldt;
+ int error;
+ struct user_desc ldt_info;
++ int oldsize, newsize;
++ struct ldt_struct *new_ldt, *old_ldt;
+
+ error = -EINVAL;
+ if (bytecount != sizeof(ldt_info))
+@@ -213,34 +230,39 @@ static int write_ldt(void __user *ptr, u
+ goto out;
+ }
+
+- mutex_lock(&mm->context.lock);
+- if (ldt_info.entry_number >= mm->context.size) {
+- error = alloc_ldt(¤t->mm->context,
+- ldt_info.entry_number + 1, 1);
+- if (error < 0)
+- goto out_unlock;
+- }
+-
+- /* Allow LDTs to be cleared by the user. */
+- if (ldt_info.base_addr == 0 && ldt_info.limit == 0) {
+- if (oldmode || LDT_empty(&ldt_info)) {
+- memset(&ldt, 0, sizeof(ldt));
+- goto install;
++ if ((oldmode && !ldt_info.base_addr && !ldt_info.limit) ||
++ LDT_empty(&ldt_info)) {
++ /* The user wants to clear the entry. */
++ memset(&ldt, 0, sizeof(ldt));
++ } else {
++ if (!IS_ENABLED(CONFIG_X86_16BIT) && !ldt_info.seg_32bit) {
++ error = -EINVAL;
++ goto out;
+ }
++
++ fill_ldt(&ldt, &ldt_info);
++ if (oldmode)
++ ldt.avl = 0;
+ }
+
+- if (!IS_ENABLED(CONFIG_X86_16BIT) && !ldt_info.seg_32bit) {
+- error = -EINVAL;
++ mutex_lock(&mm->context.lock);
++
++ old_ldt = mm->context.ldt;
++ oldsize = old_ldt ? old_ldt->size : 0;
++ newsize = max((int)(ldt_info.entry_number + 1), oldsize);
++
++ error = -ENOMEM;
++ new_ldt = alloc_ldt_struct(newsize);
++ if (!new_ldt)
+ goto out_unlock;
+- }
+
+- fill_ldt(&ldt, &ldt_info);
+- if (oldmode)
+- ldt.avl = 0;
+-
+- /* Install the new entry ... */
+-install:
+- write_ldt_entry(mm->context.ldt, ldt_info.entry_number, &ldt);
++ if (old_ldt)
++ memcpy(new_ldt->entries, old_ldt->entries, oldsize * LDT_ENTRY_SIZE);
++ new_ldt->entries[ldt_info.entry_number] = ldt;
++ finalize_ldt_struct(new_ldt);
++
++ install_ldt(mm, new_ldt);
++ free_ldt_struct(old_ldt);
+ error = 0;
+
+ out_unlock:
+--- a/arch/x86/kernel/process_64.c
++++ b/arch/x86/kernel/process_64.c
+@@ -122,11 +122,11 @@ void __show_regs(struct pt_regs *regs, i
+ void release_thread(struct task_struct *dead_task)
+ {
+ if (dead_task->mm) {
+- if (dead_task->mm->context.size) {
++ if (dead_task->mm->context.ldt) {
+ pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
+ dead_task->comm,
+ dead_task->mm->context.ldt,
+- dead_task->mm->context.size);
++ dead_task->mm->context.ldt->size);
+ BUG();
+ }
+ }
+--- a/arch/x86/kernel/step.c
++++ b/arch/x86/kernel/step.c
+@@ -5,6 +5,7 @@
+ #include <linux/mm.h>
+ #include <linux/ptrace.h>
+ #include <asm/desc.h>
++#include <asm/mmu_context.h>
+
+ unsigned long convert_ip_to_linear(struct task_struct *child, struct pt_regs *regs)
+ {
+@@ -30,10 +31,11 @@ unsigned long convert_ip_to_linear(struc
+ seg &= ~7UL;
+
+ mutex_lock(&child->mm->context.lock);
+- if (unlikely((seg >> 3) >= child->mm->context.size))
++ if (unlikely(!child->mm->context.ldt ||
++ (seg >> 3) >= child->mm->context.ldt->size))
+ addr = -1L; /* bogus selector, access would fault */
+ else {
+- desc = child->mm->context.ldt + seg;
++ desc = &child->mm->context.ldt->entries[seg];
+ base = get_desc_base(desc);
+
+ /* 16-bit code segment? */
+--- a/arch/x86/power/cpu.c
++++ b/arch/x86/power/cpu.c
+@@ -23,6 +23,7 @@
+ #include <asm/debugreg.h>
+ #include <asm/fpu-internal.h> /* pcntxt_mask */
+ #include <asm/cpu.h>
++#include <asm/mmu_context.h>
+
+ #ifdef CONFIG_X86_32
+ __visible unsigned long saved_context_ebx;
+@@ -154,7 +155,7 @@ static void fix_processor_context(void)
+ syscall_init(); /* This sets MSR_*STAR and related */
+ #endif
+ load_TR_desc(); /* This does ltr */
+- load_LDT(¤t->active_mm->context); /* This does lldt */
++ load_mm_ldt(current->active_mm); /* This does lldt */
+ }
+
+ /**
projects / thirdparty / kernel / stable-queue.git / commitdiff
