x86/kprobes: Avoid kretprobe recursion bug

[thirdparty/kernel/linux.git] / arch / x86 / kernel / kprobes / core.c

diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index a034cb808e7eb482e6fd8eae3fac9afca63b429c..fed46ddb1eef2d3de307f1cbb899f45c4b3e67c2 100644 (file)
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -569,6 +569,7 @@ void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
        unsigned long *sara = stack_addr(regs);
 
        ri->ret_addr = (kprobe_opcode_t *) *sara;
+       ri->fp = sara;
 
        /* Replace the return addr with trampoline addr */
        *sara = (unsigned long) &kretprobe_trampoline;
@@ -748,26 +749,48 @@ asm(
 NOKPROBE_SYMBOL(kretprobe_trampoline);
 STACK_FRAME_NON_STANDARD(kretprobe_trampoline);
 
+static struct kprobe kretprobe_kprobe = {
+       .addr = (void *)kretprobe_trampoline,
+};
+
 /*
  * Called from kretprobe_trampoline
  */
 static __used void *trampoline_handler(struct pt_regs *regs)
 {
+       struct kprobe_ctlblk *kcb;
        struct kretprobe_instance *ri = NULL;
        struct hlist_head *head, empty_rp;
        struct hlist_node *tmp;
        unsigned long flags, orig_ret_address = 0;
        unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
        kprobe_opcode_t *correct_ret_addr = NULL;
+       void *frame_pointer;
+       bool skipped = false;
+
+       preempt_disable();
+
+       /*
+        * Set a dummy kprobe for avoiding kretprobe recursion.
+        * Since kretprobe never run in kprobe handler, kprobe must not
+        * be running at this point.
+        */
+       kcb = get_kprobe_ctlblk();
+       __this_cpu_write(current_kprobe, &kretprobe_kprobe);
+       kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 
        INIT_HLIST_HEAD(&empty_rp);
        kretprobe_hash_lock(current, &head, &flags);
        /* fixup registers */
 #ifdef CONFIG_X86_64
        regs->cs = __KERNEL_CS;
+       /* On x86-64, we use pt_regs->sp for return address holder. */
+       frame_pointer = &regs->sp;
 #else
        regs->cs = __KERNEL_CS | get_kernel_rpl();
        regs->gs = 0;
+       /* On x86-32, we use pt_regs->flags for return address holder. */
+       frame_pointer = &regs->flags;
 #endif
        regs->ip = trampoline_address;
        regs->orig_ax = ~0UL;
@@ -789,8 +812,25 @@ static __used void *trampoline_handler(struct pt_regs *regs)
                if (ri->task != current)
                        /* another task is sharing our hash bucket */
                        continue;
+               /*
+                * Return probes must be pushed on this hash list correct
+                * order (same as return order) so that it can be poped
+                * correctly. However, if we find it is pushed it incorrect
+                * order, this means we find a function which should not be
+                * probed, because the wrong order entry is pushed on the
+                * path of processing other kretprobe itself.
+                */
+               if (ri->fp != frame_pointer) {
+                       if (!skipped)
+                               pr_warn("kretprobe is stacked incorrectly. Trying to fixup.\n");
+                       skipped = true;
+                       continue;
+               }
 
                orig_ret_address = (unsigned long)ri->ret_addr;
+               if (skipped)
+                       pr_warn("%ps must be blacklisted because of incorrect kretprobe order\n",
+                               ri->rp->kp.addr);
 
                if (orig_ret_address != trampoline_address)
                        /*
@@ -808,14 +848,15 @@ static __used void *trampoline_handler(struct pt_regs *regs)
                if (ri->task != current)
                        /* another task is sharing our hash bucket */
                        continue;
+               if (ri->fp != frame_pointer)
+                       continue;
 
                orig_ret_address = (unsigned long)ri->ret_addr;
                if (ri->rp && ri->rp->handler) {
                        __this_cpu_write(current_kprobe, &ri->rp->kp);
-                       get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
                        ri->ret_addr = correct_ret_addr;
                        ri->rp->handler(ri, regs);
-                       __this_cpu_write(current_kprobe, NULL);
+                       __this_cpu_write(current_kprobe, &kretprobe_kprobe);
                }
 
                recycle_rp_inst(ri, &empty_rp);
@@ -831,6 +872,9 @@ static __used void *trampoline_handler(struct pt_regs *regs)
 
        kretprobe_hash_unlock(current, &flags);
 
+       __this_cpu_write(current_kprobe, NULL);
+       preempt_enable();
+
        hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
                hlist_del(&ri->hlist);
                kfree(ri);