[thirdparty/linux.git] / arch / powerpc / kernel / hw_breakpoint.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
 * using the CPU's debug registers. Derived from
 * "arch/x86/kernel/hw_breakpoint.c"
 *
 * Copyright 2010 IBM Corporation
 * Author: K.Prasad <prasad@linux.vnet.ibm.com>
 */

#include <linux/hw_breakpoint.h>
#include <linux/notifier.h>
#include <linux/kprobes.h>
#include <linux/percpu.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/debugfs.h>
#include <linux/init.h>

#include <asm/hw_breakpoint.h>
#include <asm/processor.h>
#include <asm/sstep.h>
#include <asm/debug.h>
#include <asm/debugfs.h>
#include <asm/hvcall.h>
#include <linux/uaccess.h>

/*
 * Stores the breakpoints currently in use on each breakpoint address
 * register for every cpu
 */
static DEFINE_PER_CPU(struct perf_event *, bp_per_reg);

/*
 * Returns total number of data or instruction breakpoints available.
 */
int hw_breakpoint_slots(int type)
{
	if (type == TYPE_DATA)
		return HBP_NUM;
	return 0;		/* no instruction breakpoints available */
}

/*
 * Install a perf counter breakpoint.
 *
 * We seek a free debug address register and use it for this
 * breakpoint.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
 */
int arch_install_hw_breakpoint(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	struct perf_event **slot = this_cpu_ptr(&bp_per_reg);

	*slot = bp;

	/*
	 * Do not install DABR values if the instruction must be single-stepped.
	 * If so, DABR will be populated in single_step_dabr_instruction().
	 */
	if (current->thread.last_hit_ubp != bp)
		__set_breakpoint(info);

	return 0;
}

/*
 * Uninstall the breakpoint contained in the given counter.
 *
 * First we search the debug address register it uses and then we disable
 * it.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
 */
void arch_uninstall_hw_breakpoint(struct perf_event *bp)
{
	struct perf_event **slot = this_cpu_ptr(&bp_per_reg);

	if (*slot != bp) {
		WARN_ONCE(1, "Can't find the breakpoint");
		return;
	}

	*slot = NULL;
	hw_breakpoint_disable();
}

/*
 * Perform cleanup of arch-specific counters during unregistration
 * of the perf-event
 */
void arch_unregister_hw_breakpoint(struct perf_event *bp)
{
	/*
	 * If the breakpoint is unregistered between a hw_breakpoint_handler()
	 * and the single_step_dabr_instruction(), then cleanup the breakpoint
	 * restoration variables to prevent dangling pointers.
	 * FIXME, this should not be using bp->ctx at all! Sayeth peterz.
	 */
	if (bp->ctx && bp->ctx->task && bp->ctx->task != ((void *)-1L))
		bp->ctx->task->thread.last_hit_ubp = NULL;
}

/*
 * Check for virtual address in kernel space.
 */
int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
{
	return is_kernel_addr(hw->address);
}

int arch_bp_generic_fields(int type, int *gen_bp_type)
{
	*gen_bp_type = 0;
	if (type & HW_BRK_TYPE_READ)
		*gen_bp_type |= HW_BREAKPOINT_R;
	if (type & HW_BRK_TYPE_WRITE)
		*gen_bp_type |= HW_BREAKPOINT_W;
	if (*gen_bp_type == 0)
		return -EINVAL;
	return 0;
}

/*
 * Validate the arch-specific HW Breakpoint register settings
 */
int hw_breakpoint_arch_parse(struct perf_event *bp,
			     const struct perf_event_attr *attr,
			     struct arch_hw_breakpoint *hw)
{
	int ret = -EINVAL, length_max;

	if (!bp)
		return ret;

	hw->type = HW_BRK_TYPE_TRANSLATE;
	if (attr->bp_type & HW_BREAKPOINT_R)
		hw->type |= HW_BRK_TYPE_READ;
	if (attr->bp_type & HW_BREAKPOINT_W)
		hw->type |= HW_BRK_TYPE_WRITE;
	if (hw->type == HW_BRK_TYPE_TRANSLATE)
		/* must set alteast read or write */
		return ret;
	if (!attr->exclude_user)
		hw->type |= HW_BRK_TYPE_USER;
	if (!attr->exclude_kernel)
		hw->type |= HW_BRK_TYPE_KERNEL;
	if (!attr->exclude_hv)
		hw->type |= HW_BRK_TYPE_HYP;
	hw->address = attr->bp_addr;
	hw->len = attr->bp_len;

	/*
	 * Since breakpoint length can be a maximum of HW_BREAKPOINT_LEN(8)
	 * and breakpoint addresses are aligned to nearest double-word
	 * HW_BREAKPOINT_ALIGN by rounding off to the lower address, the
	 * 'symbolsize' should satisfy the check below.
	 */
	if (!ppc_breakpoint_available())
		return -ENODEV;
	length_max = 8; /* DABR */
	if (dawr_enabled()) {
		length_max = 512 ; /* 64 doublewords */
		/* DAWR region can't cross 512 boundary */
		if ((attr->bp_addr >> 9) !=
		    ((attr->bp_addr + attr->bp_len - 1) >> 9))
			return -EINVAL;
	}
	if (hw->len >
	    (length_max - (hw->address & HW_BREAKPOINT_ALIGN)))
		return -EINVAL;
	return 0;
}

/*
 * Restores the breakpoint on the debug registers.
 * Invoke this function if it is known that the execution context is
 * about to change to cause loss of MSR_SE settings.
 */
void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs)
{
	struct arch_hw_breakpoint *info;

	if (likely(!tsk->thread.last_hit_ubp))
		return;

	info = counter_arch_bp(tsk->thread.last_hit_ubp);
	regs->msr &= ~MSR_SE;
	__set_breakpoint(info);
	tsk->thread.last_hit_ubp = NULL;
}

/*
 * Handle debug exception notifications.
 */
int hw_breakpoint_handler(struct die_args *args)
{
	int rc = NOTIFY_STOP;
	struct perf_event *bp;
	struct pt_regs *regs = args->regs;
#ifndef CONFIG_PPC_8xx
	int stepped = 1;
	unsigned int instr;
#endif
	struct arch_hw_breakpoint *info;
	unsigned long dar = regs->dar;

	/* Disable breakpoints during exception handling */
	hw_breakpoint_disable();

	/*
	 * The counter may be concurrently released but that can only
	 * occur from a call_rcu() path. We can then safely fetch
	 * the breakpoint, use its callback, touch its counter
	 * while we are in an rcu_read_lock() path.
	 */
	rcu_read_lock();

	bp = __this_cpu_read(bp_per_reg);
	if (!bp) {
		rc = NOTIFY_DONE;
		goto out;
	}
	info = counter_arch_bp(bp);

	/*
	 * Return early after invoking user-callback function without restoring
	 * DABR if the breakpoint is from ptrace which always operates in
	 * one-shot mode. The ptrace-ed process will receive the SIGTRAP signal
	 * generated in do_dabr().
	 */
	if (bp->overflow_handler == ptrace_triggered) {
		perf_bp_event(bp, regs);
		rc = NOTIFY_DONE;
		goto out;
	}

	/*
	 * Verify if dar lies within the address range occupied by the symbol
	 * being watched to filter extraneous exceptions.  If it doesn't,
	 * we still need to single-step the instruction, but we don't
	 * generate an event.
	 */
	info->type &= ~HW_BRK_TYPE_EXTRANEOUS_IRQ;
	if (!((bp->attr.bp_addr <= dar) &&
	      (dar - bp->attr.bp_addr < bp->attr.bp_len)))
		info->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ;

#ifndef CONFIG_PPC_8xx
	/* Do not emulate user-space instructions, instead single-step them */
	if (user_mode(regs)) {
		current->thread.last_hit_ubp = bp;
		regs->msr |= MSR_SE;
		goto out;
	}

	stepped = 0;
	instr = 0;
	if (!__get_user_inatomic(instr, (unsigned int *) regs->nip))
		stepped = emulate_step(regs, instr);

	/*
	 * emulate_step() could not execute it. We've failed in reliably
	 * handling the hw-breakpoint. Unregister it and throw a warning
	 * message to let the user know about it.
	 */
	if (!stepped) {
		WARN(1, "Unable to handle hardware breakpoint. Breakpoint at "
			"0x%lx will be disabled.", info->address);
		perf_event_disable_inatomic(bp);
		goto out;
	}
#endif
	/*
	 * As a policy, the callback is invoked in a 'trigger-after-execute'
	 * fashion
	 */
	if (!(info->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
		perf_bp_event(bp, regs);

	__set_breakpoint(info);
out:
	rcu_read_unlock();
	return rc;
}
NOKPROBE_SYMBOL(hw_breakpoint_handler);

/*
 * Handle single-step exceptions following a DABR hit.
 */
static int single_step_dabr_instruction(struct die_args *args)
{
	struct pt_regs *regs = args->regs;
	struct perf_event *bp = NULL;
	struct arch_hw_breakpoint *info;

	bp = current->thread.last_hit_ubp;
	/*
	 * Check if we are single-stepping as a result of a
	 * previous HW Breakpoint exception
	 */
	if (!bp)
		return NOTIFY_DONE;

	info = counter_arch_bp(bp);

	/*
	 * We shall invoke the user-defined callback function in the single
	 * stepping handler to confirm to 'trigger-after-execute' semantics
	 */
	if (!(info->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
		perf_bp_event(bp, regs);

	__set_breakpoint(info);
	current->thread.last_hit_ubp = NULL;

	/*
	 * If the process was being single-stepped by ptrace, let the
	 * other single-step actions occur (e.g. generate SIGTRAP).
	 */
	if (test_thread_flag(TIF_SINGLESTEP))
		return NOTIFY_DONE;

	return NOTIFY_STOP;
}
NOKPROBE_SYMBOL(single_step_dabr_instruction);

/*
 * Handle debug exception notifications.
 */
int hw_breakpoint_exceptions_notify(
		struct notifier_block *unused, unsigned long val, void *data)
{
	int ret = NOTIFY_DONE;

	switch (val) {
	case DIE_DABR_MATCH:
		ret = hw_breakpoint_handler(data);
		break;
	case DIE_SSTEP:
		ret = single_step_dabr_instruction(data);
		break;
	}

	return ret;
}
NOKPROBE_SYMBOL(hw_breakpoint_exceptions_notify);

/*
 * Release the user breakpoints used by ptrace
 */
void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
{
	struct thread_struct *t = &tsk->thread;

	unregister_hw_breakpoint(t->ptrace_bps[0]);
	t->ptrace_bps[0] = NULL;
}

void hw_breakpoint_pmu_read(struct perf_event *bp)
{
	/* TODO */
}

bool dawr_force_enable;
EXPORT_SYMBOL_GPL(dawr_force_enable);

static ssize_t dawr_write_file_bool(struct file *file,
				    const char __user *user_buf,
				    size_t count, loff_t *ppos)
{
	struct arch_hw_breakpoint null_brk = {0, 0, 0};
	size_t rc;

	/* Send error to user if they hypervisor won't allow us to write DAWR */
	if ((!dawr_force_enable) &&
	    (firmware_has_feature(FW_FEATURE_LPAR)) &&
	    (set_dawr(&null_brk) != H_SUCCESS))
		return -1;

	rc = debugfs_write_file_bool(file, user_buf, count, ppos);
	if (rc)
		return rc;

	/* If we are clearing, make sure all CPUs have the DAWR cleared */
	if (!dawr_force_enable)
		smp_call_function((smp_call_func_t)set_dawr, &null_brk, 0);

	return rc;
}

static const struct file_operations dawr_enable_fops = {
	.read =		debugfs_read_file_bool,
	.write =	dawr_write_file_bool,
	.open =		simple_open,
	.llseek =	default_llseek,
};

static int __init dawr_force_setup(void)
{
	dawr_force_enable = false;

	if (cpu_has_feature(CPU_FTR_DAWR)) {
		/* Don't setup sysfs file for user control on P8 */
		dawr_force_enable = true;
		return 0;
	}

	if (PVR_VER(mfspr(SPRN_PVR)) == PVR_POWER9) {
		/* Turn DAWR off by default, but allow admin to turn it on */
		dawr_force_enable = false;
		debugfs_create_file_unsafe("dawr_enable_dangerous", 0600,
					   powerpc_debugfs_root,
					   &dawr_force_enable,
					   &dawr_enable_fops);
	}
	return 0;
}
arch_initcall(dawr_force_setup);
Commit	Line	Data
1a59d1b8	1	// SPDX-License-Identifier: GPL-2.0-or-later
5aae8a53 P	2	/*
	3	* HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
	4	* using the CPU's debug registers. Derived from
	5	* "arch/x86/kernel/hw_breakpoint.c"
	6	*
5aae8a53 P	7	* Copyright 2010 IBM Corporation
5aae8a53 P	8	* Author: K.Prasad <prasad@linux.vnet.ibm.com>
5aae8a53 P	9	*/
	10
	11	#include <linux/hw_breakpoint.h>
	12	#include <linux/notifier.h>
	13	#include <linux/kprobes.h>
	14	#include <linux/percpu.h>
	15	#include <linux/kernel.h>
5aae8a53	16	#include <linux/sched.h>
5aae8a53	17	#include <linux/smp.h>
c1fe190c MN	18	#include <linux/debugfs.h>
c1fe190c MN	19	#include <linux/init.h>
5aae8a53 P	20
	21	#include <asm/hw_breakpoint.h>
	22	#include <asm/processor.h>
	23	#include <asm/sstep.h>
85ce9a5d	24	#include <asm/debug.h>
c1fe190c MN	25	#include <asm/debugfs.h>
c1fe190c MN	26	#include <asm/hvcall.h>
7c0f6ba6	27	#include <linux/uaccess.h>
5aae8a53 P	28
	29	/*
	30	* Stores the breakpoints currently in use on each breakpoint address
	31	* register for every cpu
	32	*/
	33	static DEFINE_PER_CPU(struct perf_event *, bp_per_reg);
	34
d09ec738 PM	35	/*
	36	* Returns total number of data or instruction breakpoints available.
	37	*/
	38	int hw_breakpoint_slots(int type)
	39	{
	40	if (type == TYPE_DATA)
	41	return HBP_NUM;
	42	return 0; /* no instruction breakpoints available */
	43	}
	44
5aae8a53 P	45	/*
	46	* Install a perf counter breakpoint.
	47	*
	48	* We seek a free debug address register and use it for this
	49	* breakpoint.
	50	*
	51	* Atomic: we hold the counter->ctx->lock and we only handle variables
	52	* and registers local to this cpu.
	53	*/
	54	int arch_install_hw_breakpoint(struct perf_event *bp)
	55	{
	56	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
69111bac	57	struct perf_event **slot = this_cpu_ptr(&bp_per_reg);
5aae8a53 P	58
	59	*slot = bp;
	60
	61	/*
	62	* Do not install DABR values if the instruction must be single-stepped.
	63	* If so, DABR will be populated in single_step_dabr_instruction().
	64	*/
	65	if (current->thread.last_hit_ubp != bp)
21f58507	66	__set_breakpoint(info);
5aae8a53 P	67
	68	return 0;
	69	}
	70
	71	/*
	72	* Uninstall the breakpoint contained in the given counter.
	73	*
	74	* First we search the debug address register it uses and then we disable
	75	* it.
	76	*
	77	* Atomic: we hold the counter->ctx->lock and we only handle variables
	78	* and registers local to this cpu.
	79	*/
	80	void arch_uninstall_hw_breakpoint(struct perf_event *bp)
	81	{
69111bac	82	struct perf_event **slot = this_cpu_ptr(&bp_per_reg);
5aae8a53 P	83
	84	if (*slot != bp) {
	85	WARN_ONCE(1, "Can't find the breakpoint");
	86	return;
	87	}
	88
	89	*slot = NULL;
9422de3e	90	hw_breakpoint_disable();
5aae8a53 P	91	}
	92
	93	/*
	94	* Perform cleanup of arch-specific counters during unregistration
	95	* of the perf-event
	96	*/
	97	void arch_unregister_hw_breakpoint(struct perf_event *bp)
	98	{
	99	/*
	100	* If the breakpoint is unregistered between a hw_breakpoint_handler()
	101	* and the single_step_dabr_instruction(), then cleanup the breakpoint
	102	* restoration variables to prevent dangling pointers.
fb822e60	103	* FIXME, this should not be using bp->ctx at all! Sayeth peterz.
5aae8a53	104	*/
fb822e60	105	if (bp->ctx && bp->ctx->task && bp->ctx->task != ((void *)-1L))
5aae8a53 P	106	bp->ctx->task->thread.last_hit_ubp = NULL;
	107	}
	108
	109	/*
	110	* Check for virtual address in kernel space.
	111	*/
8e983ff9	112	int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
5aae8a53	113	{
8e983ff9	114	return is_kernel_addr(hw->address);
5aae8a53 P	115	}
	116
	117	int arch_bp_generic_fields(int type, int *gen_bp_type)
	118	{
9422de3e MN	119	*gen_bp_type = 0;
	120	if (type & HW_BRK_TYPE_READ)
	121	*gen_bp_type \|= HW_BREAKPOINT_R;
	122	if (type & HW_BRK_TYPE_WRITE)
	123	*gen_bp_type \|= HW_BREAKPOINT_W;
	124	if (*gen_bp_type == 0)
5aae8a53	125	return -EINVAL;
5aae8a53 P	126	return 0;
	127	}
	128
	129	/*
	130	* Validate the arch-specific HW Breakpoint register settings
	131	*/
5d5176ba FW	132	int hw_breakpoint_arch_parse(struct perf_event *bp,
	133	const struct perf_event_attr *attr,
	134	struct arch_hw_breakpoint *hw)
5aae8a53	135	{
4ae7ebe9	136	int ret = -EINVAL, length_max;
5aae8a53 P	137
	138	if (!bp)
	139	return ret;
	140
5d5176ba FW	141	hw->type = HW_BRK_TYPE_TRANSLATE;
	142	if (attr->bp_type & HW_BREAKPOINT_R)
	143	hw->type \|= HW_BRK_TYPE_READ;
	144	if (attr->bp_type & HW_BREAKPOINT_W)
	145	hw->type \|= HW_BRK_TYPE_WRITE;
	146	if (hw->type == HW_BRK_TYPE_TRANSLATE)
9422de3e	147	/* must set alteast read or write */
5aae8a53	148	return ret;
5d5176ba FW	149	if (!attr->exclude_user)
	150	hw->type \|= HW_BRK_TYPE_USER;
	151	if (!attr->exclude_kernel)
	152	hw->type \|= HW_BRK_TYPE_KERNEL;
	153	if (!attr->exclude_hv)
	154	hw->type \|= HW_BRK_TYPE_HYP;
	155	hw->address = attr->bp_addr;
	156	hw->len = attr->bp_len;
5aae8a53 P	157
	158	/*
	159	* Since breakpoint length can be a maximum of HW_BREAKPOINT_LEN(8)
	160	* and breakpoint addresses are aligned to nearest double-word
	161	* HW_BREAKPOINT_ALIGN by rounding off to the lower address, the
	162	* 'symbolsize' should satisfy the check below.
	163	*/
85ce9a5d MN	164	if (!ppc_breakpoint_available())
85ce9a5d MN	165	return -ENODEV;
4ae7ebe9	166	length_max = 8; /* DABR */
c1fe190c	167	if (dawr_enabled()) {
4ae7ebe9 MN	168	length_max = 512 ; /* 64 doublewords */
4ae7ebe9 MN	169	/* DAWR region can't cross 512 boundary */
5d5176ba FW	170	if ((attr->bp_addr >> 9) !=
5d5176ba FW	171	((attr->bp_addr + attr->bp_len - 1) >> 9))
4ae7ebe9 MN	172	return -EINVAL;
4ae7ebe9 MN	173	}
5d5176ba FW	174	if (hw->len >
5d5176ba FW	175	(length_max - (hw->address & HW_BREAKPOINT_ALIGN)))
5aae8a53 P	176	return -EINVAL;
	177	return 0;
	178	}
	179
06532a67 P	180	/*
	181	* Restores the breakpoint on the debug registers.
	182	* Invoke this function if it is known that the execution context is
	183	* about to change to cause loss of MSR_SE settings.
	184	*/
	185	void thread_change_pc(struct task_struct tsk, struct pt_regs regs)
	186	{
	187	struct arch_hw_breakpoint *info;
	188
	189	if (likely(!tsk->thread.last_hit_ubp))
	190	return;
	191
	192	info = counter_arch_bp(tsk->thread.last_hit_ubp);
	193	regs->msr &= ~MSR_SE;
21f58507	194	__set_breakpoint(info);
06532a67 P	195	tsk->thread.last_hit_ubp = NULL;
	196	}
	197
5aae8a53 P	198	/*
	199	* Handle debug exception notifications.
	200	*/
03465f89	201	int hw_breakpoint_handler(struct die_args *args)
5aae8a53	202	{
5aae8a53 P	203	int rc = NOTIFY_STOP;
	204	struct perf_event *bp;
	205	struct pt_regs *regs = args->regs;
4ad8622d	206	#ifndef CONFIG_PPC_8xx
5aae8a53	207	int stepped = 1;
5aae8a53	208	unsigned int instr;
4ad8622d CL	209	#endif
4ad8622d CL	210	struct arch_hw_breakpoint *info;
e3e94084	211	unsigned long dar = regs->dar;
5aae8a53 P	212
5aae8a53 P	213	/* Disable breakpoints during exception handling */
9422de3e	214	hw_breakpoint_disable();
574cb248	215
5aae8a53 P	216	/*
	217	* The counter may be concurrently released but that can only
	218	* occur from a call_rcu() path. We can then safely fetch
	219	* the breakpoint, use its callback, touch its counter
	220	* while we are in an rcu_read_lock() path.
	221	*/
	222	rcu_read_lock();
	223
69111bac	224	bp = __this_cpu_read(bp_per_reg);
c21a493a RB	225	if (!bp) {
c21a493a RB	226	rc = NOTIFY_DONE;
5aae8a53	227	goto out;
c21a493a	228	}
5aae8a53	229	info = counter_arch_bp(bp);
5aae8a53 P	230
	231	/*
	232	* Return early after invoking user-callback function without restoring
	233	* DABR if the breakpoint is from ptrace which always operates in
	234	* one-shot mode. The ptrace-ed process will receive the SIGTRAP signal
	235	* generated in do_dabr().
	236	*/
574cb248	237	if (bp->overflow_handler == ptrace_triggered) {
5aae8a53 P	238	perf_bp_event(bp, regs);
	239	rc = NOTIFY_DONE;
	240	goto out;
	241	}
	242
e3e94084 P	243	/*
e3e94084 P	244	* Verify if dar lies within the address range occupied by the symbol
574cb248 PM	245	* being watched to filter extraneous exceptions. If it doesn't,
	246	* we still need to single-step the instruction, but we don't
	247	* generate an event.
e3e94084	248	*/
540e07c6	249	info->type &= ~HW_BRK_TYPE_EXTRANEOUS_IRQ;
9422de3e MN	250	if (!((bp->attr.bp_addr <= dar) &&
	251	(dar - bp->attr.bp_addr < bp->attr.bp_len)))
	252	info->type \|= HW_BRK_TYPE_EXTRANEOUS_IRQ;
e3e94084	253
4ad8622d	254	#ifndef CONFIG_PPC_8xx
5aae8a53 P	255	/* Do not emulate user-space instructions, instead single-step them */
5aae8a53 P	256	if (user_mode(regs)) {
6d9c00c6	257	current->thread.last_hit_ubp = bp;
5aae8a53 P	258	regs->msr \|= MSR_SE;
	259	goto out;
	260	}
	261
	262	stepped = 0;
	263	instr = 0;
	264	if (!__get_user_inatomic(instr, (unsigned int *) regs->nip))
	265	stepped = emulate_step(regs, instr);
	266
	267	/*
	268	* emulate_step() could not execute it. We've failed in reliably
	269	* handling the hw-breakpoint. Unregister it and throw a warning
	270	* message to let the user know about it.
	271	*/
	272	if (!stepped) {
	273	WARN(1, "Unable to handle hardware breakpoint. Breakpoint at "
	274	"0x%lx will be disabled.", info->address);
5aab90ce	275	perf_event_disable_inatomic(bp);
5aae8a53 P	276	goto out;
5aae8a53 P	277	}
4ad8622d	278	#endif
5aae8a53 P	279	/*
	280	* As a policy, the callback is invoked in a 'trigger-after-execute'
	281	* fashion
	282	*/
9422de3e	283	if (!(info->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
e3e94084	284	perf_bp_event(bp, regs);
5aae8a53	285
21f58507	286	__set_breakpoint(info);
5aae8a53 P	287	out:
	288	rcu_read_unlock();
	289	return rc;
	290	}
03465f89	291	NOKPROBE_SYMBOL(hw_breakpoint_handler);
5aae8a53 P	292
	293	/*
	294	* Handle single-step exceptions following a DABR hit.
	295	*/
03465f89	296	static int single_step_dabr_instruction(struct die_args *args)
5aae8a53 P	297	{
	298	struct pt_regs *regs = args->regs;
	299	struct perf_event *bp = NULL;
3f4693ee	300	struct arch_hw_breakpoint *info;
5aae8a53 P	301
	302	bp = current->thread.last_hit_ubp;
	303	/*
	304	* Check if we are single-stepping as a result of a
	305	* previous HW Breakpoint exception
	306	*/
	307	if (!bp)
	308	return NOTIFY_DONE;
	309
3f4693ee	310	info = counter_arch_bp(bp);
5aae8a53 P	311
	312	/*
	313	* We shall invoke the user-defined callback function in the single
	314	* stepping handler to confirm to 'trigger-after-execute' semantics
	315	*/
9422de3e	316	if (!(info->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
e3e94084	317	perf_bp_event(bp, regs);
5aae8a53	318
21f58507	319	__set_breakpoint(info);
76b0f133 PM	320	current->thread.last_hit_ubp = NULL;
76b0f133 PM	321
5aae8a53	322	/*
76b0f133 PM	323	* If the process was being single-stepped by ptrace, let the
76b0f133 PM	324	* other single-step actions occur (e.g. generate SIGTRAP).
5aae8a53	325	*/
76b0f133 PM	326	if (test_thread_flag(TIF_SINGLESTEP))
76b0f133 PM	327	return NOTIFY_DONE;
5aae8a53	328
5aae8a53 P	329	return NOTIFY_STOP;
5aae8a53 P	330	}
03465f89	331	NOKPROBE_SYMBOL(single_step_dabr_instruction);
5aae8a53 P	332
	333	/*
	334	* Handle debug exception notifications.
	335	*/
03465f89	336	int hw_breakpoint_exceptions_notify(
5aae8a53 P	337	struct notifier_block unused, unsigned long val, void data)
	338	{
	339	int ret = NOTIFY_DONE;
	340
	341	switch (val) {
	342	case DIE_DABR_MATCH:
	343	ret = hw_breakpoint_handler(data);
	344	break;
	345	case DIE_SSTEP:
	346	ret = single_step_dabr_instruction(data);
	347	break;
	348	}
	349
	350	return ret;
	351	}
03465f89	352	NOKPROBE_SYMBOL(hw_breakpoint_exceptions_notify);
5aae8a53 P	353
	354	/*
	355	* Release the user breakpoints used by ptrace
	356	*/
	357	void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
	358	{
	359	struct thread_struct *t = &tsk->thread;
	360
	361	unregister_hw_breakpoint(t->ptrace_bps[0]);
	362	t->ptrace_bps[0] = NULL;
	363	}
	364
	365	void hw_breakpoint_pmu_read(struct perf_event *bp)
	366	{
	367	/* TODO */
	368	}
c1fe190c MN	369
	370	bool dawr_force_enable;
	371	EXPORT_SYMBOL_GPL(dawr_force_enable);
	372
	373	static ssize_t dawr_write_file_bool(struct file *file,
	374	const char __user *user_buf,
	375	size_t count, loff_t *ppos)
	376	{
	377	struct arch_hw_breakpoint null_brk = {0, 0, 0};
	378	size_t rc;
	379
	380	/* Send error to user if they hypervisor won't allow us to write DAWR */
	381	if ((!dawr_force_enable) &&
	382	(firmware_has_feature(FW_FEATURE_LPAR)) &&
	383	(set_dawr(&null_brk) != H_SUCCESS))
	384	return -1;
	385
	386	rc = debugfs_write_file_bool(file, user_buf, count, ppos);
	387	if (rc)
	388	return rc;
	389
	390	/* If we are clearing, make sure all CPUs have the DAWR cleared */
	391	if (!dawr_force_enable)
	392	smp_call_function((smp_call_func_t)set_dawr, &null_brk, 0);
	393
	394	return rc;
	395	}
	396
	397	static const struct file_operations dawr_enable_fops = {
	398	.read = debugfs_read_file_bool,
	399	.write = dawr_write_file_bool,
	400	.open = simple_open,
	401	.llseek = default_llseek,
	402	};
	403
	404	static int __init dawr_force_setup(void)
	405	{
	406	dawr_force_enable = false;
	407
	408	if (cpu_has_feature(CPU_FTR_DAWR)) {
	409	/* Don't setup sysfs file for user control on P8 */
	410	dawr_force_enable = true;
	411	return 0;
	412	}
	413
	414	if (PVR_VER(mfspr(SPRN_PVR)) == PVR_POWER9) {
	415	/* Turn DAWR off by default, but allow admin to turn it on */
	416	dawr_force_enable = false;
	417	debugfs_create_file_unsafe("dawr_enable_dangerous", 0600,
	418	powerpc_debugfs_root,
	419	&dawr_force_enable,
	420	&dawr_enable_fops);
	421	}
	422	return 0;
	423	}
	424	arch_initcall(dawr_force_setup);