ARM: fix branch predictor hardening

__do_user_fault() may be called with indeterminent interrupt enable
state, which means we may be preemptive at this point. This causes
problems when calling harden_branch_predictor(). For example, when
called from a data abort, do_alignment_fault()->do_bad_area().

Move harden_branch_predictor() out of __do_user_fault() and into the
calling contexts.

Moving it into do_kernel_address_page_fault(), we can be sure that
interrupts will be disabled here.

Converting do_translation_fault() to use do_kernel_address_page_fault()
rather than do_bad_area() means that we keep branch predictor handling
for translation faults. Interrupts will also be disabled at this call
site.

do_sect_fault() needs special handling, so detect user mode accesses
to kernel-addresses, and add an explicit call to branch predictor
hardening.

Finally, add branch predictor hardening to do_alignment() for the
faulting case (user mode accessing kernel addresses) before interrupts
are enabled.

This should cover all cases where harden_branch_predictor() is called,
ensuring that it is always has interrupts disabled, also ensuring that
it is called early in each call path.

Reviewed-by: Xie Yuanbin <xieyuanbin1@huawei.com>
Tested-by: Xie Yuanbin <xieyuanbin1@huawei.com>
Signed-off-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>

diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index 3c6ddb1afdc463b393447969bfadb77eb99d3483..812380f30ae36a3264542a8bd3f55ea64a7fbd73 100644 (file)
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -19,10 +19,11 @@
 #include <linux/init.h>
 #include <linux/sched/signal.h>
 #include <linux/uaccess.h>
+#include <linux/unaligned.h>
 
 #include <asm/cp15.h>
 #include <asm/system_info.h>
-#include <linux/unaligned.h>
+#include <asm/system_misc.h>
 #include <asm/opcodes.h>
 
 #include "fault.h"
@@ -809,6 +810,9 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
        int thumb2_32b = 0;
        int fault;
 
+       if (addr >= TASK_SIZE && user_mode(regs))
+               harden_branch_predictor();
+
        if (interrupts_enabled(regs))
                local_irq_enable();
 

diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 0e5b4bc7b217603d3a6c7cd494117d3c7c250b37..ed4330cc3f4e6ff2217c9715dd94ac6cb70b14f8 100644 (file)
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -198,9 +198,6 @@ __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig,
 {
        struct task_struct *tsk = current;
 
-       if (addr > TASK_SIZE)
-               harden_branch_predictor();
-
 #ifdef CONFIG_DEBUG_USER
        if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
            ((user_debug & UDBG_BUS)  && (sig == SIGBUS))) {
@@ -269,8 +266,10 @@ do_kernel_address_page_fault(struct mm_struct *mm, unsigned long addr,
                /*
                 * Fault from user mode for a kernel space address. User mode
                 * should not be faulting in kernel space, which includes the
-                * vector/khelper page. Send a SIGSEGV.
+                * vector/khelper page. Handle the branch predictor hardening
+                * while interrupts are still disabled, then send a SIGSEGV.
                 */
+               harden_branch_predictor();
                __do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
        } else {
                /*
@@ -485,16 +484,20 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
  * We enter here because the first level page table doesn't contain
  * a valid entry for the address.
  *
- * If the address is in kernel space (>= TASK_SIZE), then we are
- * probably faulting in the vmalloc() area.
+ * If this is a user address (addr < TASK_SIZE), we handle this as a
+ * normal page fault. This leaves the remainder of the function to handle
+ * kernel address translation faults.
  *
- * If the init_task's first level page tables contains the relevant
- * entry, we copy the it to this task.  If not, we send the process
- * a signal, fixup the exception, or oops the kernel.
+ * Since user mode is not permitted to access kernel addresses, pass these
+ * directly to do_kernel_address_page_fault() to handle.
  *
- * NOTE! We MUST NOT take any locks for this case. We may be in an
- * interrupt or a critical region, and should only copy the information
- * from the master page table, nothing more.
+ * Otherwise, we're probably faulting in the vmalloc() area, so try to fix
+ * that up. Note that we must not take any locks or enable interrupts in
+ * this case.
+ *
+ * If vmalloc() fixup fails, that means the non-leaf page tables did not
+ * contain an entry for this address, so handle this via
+ * do_kernel_address_page_fault().
  */
 #ifdef CONFIG_MMU
 static int __kprobes
@@ -560,7 +563,8 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
        return 0;
 
 bad_area:
-       do_bad_area(addr, fsr, regs);
+       do_kernel_address_page_fault(current->mm, addr, fsr, regs);
+
        return 0;
 }
 #else                                  /* CONFIG_MMU */
@@ -580,7 +584,16 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 static int
 do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
+       /*
+        * If this is a kernel address, but from user mode, then userspace
+        * is trying bad stuff. Invoke the branch predictor handling.
+        * Interrupts are disabled here.
+        */
+       if (addr >= TASK_SIZE && user_mode(regs))
+               harden_branch_predictor();
+
        do_bad_area(addr, fsr, regs);
+
        return 0;
 }
 #endif /* CONFIG_ARM_LPAE */