[thirdparty/linux.git] / mm / ptdump.c

// SPDX-License-Identifier: GPL-2.0

#include <linux/pagewalk.h>
#include <linux/debugfs.h>
#include <linux/ptdump.h>
#include <linux/kasan.h>

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
/*
 * This is an optimization for KASAN=y case. Since all kasan page tables
 * eventually point to the kasan_early_shadow_page we could call note_page()
 * right away without walking through lower level page tables. This saves
 * us dozens of seconds (minutes for 5-level config) while checking for
 * W+X mapping or reading kernel_page_tables debugfs file.
 */
static inline int note_kasan_page_table(struct mm_walk *walk,
					unsigned long addr)
{
	struct ptdump_state *st = walk->private;

	st->note_page_pte(st, addr, kasan_early_shadow_pte[0]);

	walk->action = ACTION_CONTINUE;

	return 0;
}
#endif

static int ptdump_pgd_entry(pgd_t *pgd, unsigned long addr,
			    unsigned long next, struct mm_walk *walk)
{
	struct ptdump_state *st = walk->private;
	pgd_t val = READ_ONCE(*pgd);

#if CONFIG_PGTABLE_LEVELS > 4 && \
		(defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS))
	if (pgd_page(val) == virt_to_page(lm_alias(kasan_early_shadow_p4d)))
		return note_kasan_page_table(walk, addr);
#endif

	if (st->effective_prot_pgd)
		st->effective_prot_pgd(st, val);

	if (pgd_leaf(val)) {
		st->note_page_pgd(st, addr, val);
		walk->action = ACTION_CONTINUE;
	}

	return 0;
}

static int ptdump_p4d_entry(p4d_t *p4d, unsigned long addr,
			    unsigned long next, struct mm_walk *walk)
{
	struct ptdump_state *st = walk->private;
	p4d_t val = READ_ONCE(*p4d);

#if CONFIG_PGTABLE_LEVELS > 3 && \
		(defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS))
	if (p4d_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pud)))
		return note_kasan_page_table(walk, addr);
#endif

	if (st->effective_prot_p4d)
		st->effective_prot_p4d(st, val);

	if (p4d_leaf(val)) {
		st->note_page_p4d(st, addr, val);
		walk->action = ACTION_CONTINUE;
	}

	return 0;
}

static int ptdump_pud_entry(pud_t *pud, unsigned long addr,
			    unsigned long next, struct mm_walk *walk)
{
	struct ptdump_state *st = walk->private;
	pud_t val = READ_ONCE(*pud);

#if CONFIG_PGTABLE_LEVELS > 2 && \
		(defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS))
	if (pud_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pmd)))
		return note_kasan_page_table(walk, addr);
#endif

	if (st->effective_prot_pud)
		st->effective_prot_pud(st, val);

	if (pud_leaf(val)) {
		st->note_page_pud(st, addr, val);
		walk->action = ACTION_CONTINUE;
	}

	return 0;
}

static int ptdump_pmd_entry(pmd_t *pmd, unsigned long addr,
			    unsigned long next, struct mm_walk *walk)
{
	struct ptdump_state *st = walk->private;
	pmd_t val = READ_ONCE(*pmd);

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
	if (pmd_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pte)))
		return note_kasan_page_table(walk, addr);
#endif

	if (st->effective_prot_pmd)
		st->effective_prot_pmd(st, val);
	if (pmd_leaf(val)) {
		st->note_page_pmd(st, addr, val);
		walk->action = ACTION_CONTINUE;
	}

	return 0;
}

static int ptdump_pte_entry(pte_t *pte, unsigned long addr,
			    unsigned long next, struct mm_walk *walk)
{
	struct ptdump_state *st = walk->private;
	pte_t val = ptep_get_lockless(pte);

	if (st->effective_prot_pte)
		st->effective_prot_pte(st, val);

	st->note_page_pte(st, addr, val);

	return 0;
}

static int ptdump_hole(unsigned long addr, unsigned long next,
		       int depth, struct mm_walk *walk)
{
	struct ptdump_state *st = walk->private;
	pte_t pte_zero = {0};
	pmd_t pmd_zero = {0};
	pud_t pud_zero = {0};
	p4d_t p4d_zero = {0};
	pgd_t pgd_zero = {0};

	switch (depth) {
	case 4:
		st->note_page_pte(st, addr, pte_zero);
		break;
	case 3:
		st->note_page_pmd(st, addr, pmd_zero);
		break;
	case 2:
		st->note_page_pud(st, addr, pud_zero);
		break;
	case 1:
		st->note_page_p4d(st, addr, p4d_zero);
		break;
	case 0:
		st->note_page_pgd(st, addr, pgd_zero);
		break;
	default:
		break;
	}
	return 0;
}

static const struct mm_walk_ops ptdump_ops = {
	.pgd_entry	= ptdump_pgd_entry,
	.p4d_entry	= ptdump_p4d_entry,
	.pud_entry	= ptdump_pud_entry,
	.pmd_entry	= ptdump_pmd_entry,
	.pte_entry	= ptdump_pte_entry,
	.pte_hole	= ptdump_hole,
};

void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd)
{
	const struct ptdump_range *range = st->range;

	mmap_write_lock(mm);
	while (range->start != range->end) {
		walk_page_range_novma(mm, range->start, range->end,
				      &ptdump_ops, pgd, st);
		range++;
	}
	mmap_write_unlock(mm);

	/* Flush out the last page */
	st->note_page_flush(st);
}

static int check_wx_show(struct seq_file *m, void *v)
{
	if (ptdump_check_wx())
		seq_puts(m, "SUCCESS\n");
	else
		seq_puts(m, "FAILED\n");

	return 0;
}

DEFINE_SHOW_ATTRIBUTE(check_wx);

static int ptdump_debugfs_init(void)
{
	debugfs_create_file("check_wx_pages", 0400, NULL, NULL, &check_wx_fops);

	return 0;
}

device_initcall(ptdump_debugfs_init);
Commit	Line	Data
30d621f6 SP	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <linux/pagewalk.h>
565474af	4	#include <linux/debugfs.h>
30d621f6 SP	5	#include <linux/ptdump.h>
	6	#include <linux/kasan.h>
	7
0fea6e9a	8	#if defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS)
30d621f6 SP	9	/*
	10	* This is an optimization for KASAN=y case. Since all kasan page tables
	11	* eventually point to the kasan_early_shadow_page we could call note_page()
	12	* right away without walking through lower level page tables. This saves
	13	* us dozens of seconds (minutes for 5-level config) while checking for
	14	* W+X mapping or reading kernel_page_tables debugfs file.
	15	*/
	16	static inline int note_kasan_page_table(struct mm_walk *walk,
	17	unsigned long addr)
	18	{
	19	struct ptdump_state *st = walk->private;
	20
e064e738	21	st->note_page_pte(st, addr, kasan_early_shadow_pte[0]);
30d621f6 SP	22
	23	walk->action = ACTION_CONTINUE;
	24
	25	return 0;
	26	}
	27	#endif
	28
	29	static int ptdump_pgd_entry(pgd_t *pgd, unsigned long addr,
	30	unsigned long next, struct mm_walk *walk)
	31	{
	32	struct ptdump_state *st = walk->private;
	33	pgd_t val = READ_ONCE(*pgd);
	34
0fea6e9a AK	35	#if CONFIG_PGTABLE_LEVELS > 4 && \
0fea6e9a AK	36	(defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS))
30d621f6 SP	37	if (pgd_page(val) == virt_to_page(lm_alias(kasan_early_shadow_p4d)))
	38	return note_kasan_page_table(walk, addr);
	39	#endif
	40
08978fc3 AK	41	if (st->effective_prot_pgd)
08978fc3 AK	42	st->effective_prot_pgd(st, val);
1494e0c3	43
d8d55f56	44	if (pgd_leaf(val)) {
e064e738	45	st->note_page_pgd(st, addr, val);
d8d55f56 MS	46	walk->action = ACTION_CONTINUE;
d8d55f56 MS	47	}
30d621f6 SP	48
	49	return 0;
	50	}
	51
	52	static int ptdump_p4d_entry(p4d_t *p4d, unsigned long addr,
	53	unsigned long next, struct mm_walk *walk)
	54	{
	55	struct ptdump_state *st = walk->private;
	56	p4d_t val = READ_ONCE(*p4d);
	57
0fea6e9a AK	58	#if CONFIG_PGTABLE_LEVELS > 3 && \
0fea6e9a AK	59	(defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS))
30d621f6 SP	60	if (p4d_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pud)))
	61	return note_kasan_page_table(walk, addr);
	62	#endif
	63
08978fc3 AK	64	if (st->effective_prot_p4d)
08978fc3 AK	65	st->effective_prot_p4d(st, val);
1494e0c3	66
d8d55f56	67	if (p4d_leaf(val)) {
e064e738	68	st->note_page_p4d(st, addr, val);
d8d55f56 MS	69	walk->action = ACTION_CONTINUE;
d8d55f56 MS	70	}
30d621f6 SP	71
	72	return 0;
	73	}
	74
	75	static int ptdump_pud_entry(pud_t *pud, unsigned long addr,
	76	unsigned long next, struct mm_walk *walk)
	77	{
	78	struct ptdump_state *st = walk->private;
	79	pud_t val = READ_ONCE(*pud);
	80
0fea6e9a AK	81	#if CONFIG_PGTABLE_LEVELS > 2 && \
0fea6e9a AK	82	(defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS))
30d621f6 SP	83	if (pud_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pmd)))
	84	return note_kasan_page_table(walk, addr);
	85	#endif
	86
08978fc3 AK	87	if (st->effective_prot_pud)
08978fc3 AK	88	st->effective_prot_pud(st, val);
1494e0c3	89
d8d55f56	90	if (pud_leaf(val)) {
e064e738	91	st->note_page_pud(st, addr, val);
d8d55f56 MS	92	walk->action = ACTION_CONTINUE;
d8d55f56 MS	93	}
30d621f6 SP	94
	95	return 0;
	96	}
	97
	98	static int ptdump_pmd_entry(pmd_t *pmd, unsigned long addr,
	99	unsigned long next, struct mm_walk *walk)
	100	{
	101	struct ptdump_state *st = walk->private;
	102	pmd_t val = READ_ONCE(*pmd);
	103
0fea6e9a	104	#if defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS)
30d621f6 SP	105	if (pmd_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pte)))
	106	return note_kasan_page_table(walk, addr);
	107	#endif
	108
08978fc3 AK	109	if (st->effective_prot_pmd)
08978fc3 AK	110	st->effective_prot_pmd(st, val);
d8d55f56	111	if (pmd_leaf(val)) {
e064e738	112	st->note_page_pmd(st, addr, val);
d8d55f56 MS	113	walk->action = ACTION_CONTINUE;
d8d55f56 MS	114	}
30d621f6 SP	115
	116	return 0;
	117	}
	118
	119	static int ptdump_pte_entry(pte_t *pte, unsigned long addr,
	120	unsigned long next, struct mm_walk *walk)
	121	{
	122	struct ptdump_state *st = walk->private;
426931e7	123	pte_t val = ptep_get_lockless(pte);
1494e0c3	124
08978fc3 AK	125	if (st->effective_prot_pte)
08978fc3 AK	126	st->effective_prot_pte(st, val);
30d621f6	127
e064e738	128	st->note_page_pte(st, addr, val);
30d621f6 SP	129
	130	return 0;
	131	}
	132
	133	static int ptdump_hole(unsigned long addr, unsigned long next,
	134	int depth, struct mm_walk *walk)
	135	{
	136	struct ptdump_state *st = walk->private;
e064e738 AK	137	pte_t pte_zero = {0};
	138	pmd_t pmd_zero = {0};
	139	pud_t pud_zero = {0};
	140	p4d_t p4d_zero = {0};
	141	pgd_t pgd_zero = {0};
	142
	143	switch (depth) {
	144	case 4:
	145	st->note_page_pte(st, addr, pte_zero);
	146	break;
	147	case 3:
	148	st->note_page_pmd(st, addr, pmd_zero);
	149	break;
	150	case 2:
	151	st->note_page_pud(st, addr, pud_zero);
	152	break;
	153	case 1:
	154	st->note_page_p4d(st, addr, p4d_zero);
	155	break;
	156	case 0:
	157	st->note_page_pgd(st, addr, pgd_zero);
	158	break;
	159	default:
	160	break;
	161	}
30d621f6 SP	162	return 0;
	163	}
	164
	165	static const struct mm_walk_ops ptdump_ops = {
	166	.pgd_entry = ptdump_pgd_entry,
	167	.p4d_entry = ptdump_p4d_entry,
	168	.pud_entry = ptdump_pud_entry,
	169	.pmd_entry = ptdump_pmd_entry,
	170	.pte_entry = ptdump_pte_entry,
	171	.pte_hole = ptdump_hole,
	172	};
	173
e47690d7	174	void ptdump_walk_pgd(struct ptdump_state st, struct mm_struct mm, pgd_t *pgd)
30d621f6 SP	175	{
	176	const struct ptdump_range *range = st->range;
	177
8782fb61	178	mmap_write_lock(mm);
30d621f6 SP	179	while (range->start != range->end) {
30d621f6 SP	180	walk_page_range_novma(mm, range->start, range->end,
e47690d7	181	&ptdump_ops, pgd, st);
30d621f6 SP	182	range++;
30d621f6 SP	183	}
8782fb61	184	mmap_write_unlock(mm);
30d621f6 SP	185
30d621f6 SP	186	/* Flush out the last page */
e064e738	187	st->note_page_flush(st);
30d621f6	188	}
565474af CL	189
	190	static int check_wx_show(struct seq_file m, void v)
	191	{
	192	if (ptdump_check_wx())
	193	seq_puts(m, "SUCCESS\n");
	194	else
	195	seq_puts(m, "FAILED\n");
	196
	197	return 0;
	198	}
	199
	200	DEFINE_SHOW_ATTRIBUTE(check_wx);
	201
	202	static int ptdump_debugfs_init(void)
	203	{
	204	debugfs_create_file("check_wx_pages", 0400, NULL, NULL, &check_wx_fops);
	205
	206	return 0;
	207	}
	208
	209	device_initcall(ptdump_debugfs_init);